@librechat/agents 3.2.68 → 3.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/agents/AgentContext.cjs +1 -1
- package/dist/cjs/common/enum.cjs +2 -0
- package/dist/cjs/common/enum.cjs.map +1 -1
- package/dist/cjs/graphs/Graph.cjs +14 -1
- package/dist/cjs/graphs/Graph.cjs.map +1 -1
- package/dist/cjs/graphs/MultiAgentGraph.cjs +1 -1
- package/dist/cjs/langfuseToolOutputTracing.cjs +4 -0
- package/dist/cjs/langfuseToolOutputTracing.cjs.map +1 -1
- package/dist/cjs/llm/openai/index.cjs +1 -1
- package/dist/cjs/main.cjs +1 -0
- package/dist/cjs/messages/format.cjs +136 -4
- package/dist/cjs/messages/format.cjs.map +1 -1
- package/dist/cjs/prompts/activityLabel.cjs +101 -0
- package/dist/cjs/prompts/activityLabel.cjs.map +1 -0
- package/dist/cjs/run.cjs +162 -1
- package/dist/cjs/run.cjs.map +1 -1
- package/dist/cjs/tools/subagent/SubagentExecutor.cjs +1 -1
- package/dist/esm/agents/AgentContext.mjs +1 -1
- package/dist/esm/common/enum.mjs +2 -0
- package/dist/esm/common/enum.mjs.map +1 -1
- package/dist/esm/graphs/Graph.mjs +15 -2
- package/dist/esm/graphs/Graph.mjs.map +1 -1
- package/dist/esm/graphs/MultiAgentGraph.mjs +1 -1
- package/dist/esm/langfuseToolOutputTracing.mjs +4 -1
- package/dist/esm/langfuseToolOutputTracing.mjs.map +1 -1
- package/dist/esm/llm/openai/index.mjs +1 -1
- package/dist/esm/main.mjs +2 -2
- package/dist/esm/messages/format.mjs +136 -5
- package/dist/esm/messages/format.mjs.map +1 -1
- package/dist/esm/prompts/activityLabel.mjs +100 -0
- package/dist/esm/prompts/activityLabel.mjs.map +1 -0
- package/dist/esm/run.mjs +163 -2
- package/dist/esm/run.mjs.map +1 -1
- package/dist/esm/tools/subagent/SubagentExecutor.mjs +1 -1
- package/dist/types/common/enum.d.ts +3 -1
- package/dist/types/langfuseToolOutputTracing.d.ts +4 -0
- package/dist/types/messages/format.d.ts +22 -0
- package/dist/types/prompts/activityLabel.d.ts +31 -0
- package/dist/types/run.d.ts +14 -0
- package/dist/types/types/activityLabel.d.ts +53 -0
- package/dist/types/types/index.d.ts +1 -0
- package/dist/types/types/stream.d.ts +2 -0
- package/package.json +1 -1
- package/src/common/enum.ts +2 -0
- package/src/graphs/Graph.ts +20 -0
- package/src/langfuseToolOutputTracing.ts +4 -1
- package/src/messages/foldToollessToolBlocks.test.ts +438 -0
- package/src/messages/format.ts +233 -5
- package/src/prompts/activityLabel.ts +177 -0
- package/src/run.ts +298 -2
- package/src/specs/activity-label-prompt.test.ts +128 -0
- package/src/specs/activity-label-trace-seed.test.ts +47 -0
- package/src/specs/bedrock-toolless.live.test.ts +123 -0
- package/src/types/activityLabel.ts +55 -0
- package/src/types/index.ts +1 -0
- package/src/types/stream.ts +2 -0
package/src/run.ts
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
// src/run.ts
|
|
2
|
-
import { HumanMessage } from '@langchain/core/messages';
|
|
3
2
|
import { PromptTemplate } from '@langchain/core/prompts';
|
|
4
3
|
import { RunnableLambda } from '@langchain/core/runnables';
|
|
5
4
|
import { AzureChatOpenAI, ChatOpenAI } from '@langchain/openai';
|
|
6
5
|
import { BaseCallbackHandler } from '@langchain/core/callbacks/base';
|
|
6
|
+
import { HumanMessage, SystemMessage } from '@langchain/core/messages';
|
|
7
7
|
import {
|
|
8
8
|
Command,
|
|
9
9
|
INTERRUPT,
|
|
@@ -26,10 +26,19 @@ import {
|
|
|
26
26
|
isLangfuseCallbackHandler,
|
|
27
27
|
withLangfuseAttributes,
|
|
28
28
|
} from '@/langfuse';
|
|
29
|
+
import {
|
|
30
|
+
hasToolOutputTracingConfig,
|
|
31
|
+
resolveLangfuseConfig,
|
|
32
|
+
resolveToolOutputTracingConfig,
|
|
33
|
+
} from '@/langfuseConfig';
|
|
29
34
|
import {
|
|
30
35
|
resolveLangfuseRuntimeScope,
|
|
31
36
|
withLangfuseRuntimeScope,
|
|
32
37
|
} from '@/langfuseRuntimeScope';
|
|
38
|
+
import {
|
|
39
|
+
ACTIVITY_LABEL_PROMPT,
|
|
40
|
+
buildActivityLabelPrompt,
|
|
41
|
+
} from '@/prompts/activityLabel';
|
|
33
42
|
import {
|
|
34
43
|
appendCallbacks,
|
|
35
44
|
findCallback,
|
|
@@ -43,7 +52,7 @@ import { createTokenCounter, encodingForModel } from '@/utils/tokens';
|
|
|
43
52
|
import { initializeLangfuseTracing } from './instrumentation';
|
|
44
53
|
import { GraphEvents, Callback, TitleMethod } from '@/common';
|
|
45
54
|
import { MultiAgentGraph } from '@/graphs/MultiAgentGraph';
|
|
46
|
-
import {
|
|
55
|
+
import { getTraceIdSeed } from '@/langfuseRuntimeContext';
|
|
47
56
|
import { StandardGraph } from '@/graphs/Graph';
|
|
48
57
|
import { initializeModel } from '@/llm/init';
|
|
49
58
|
import { HandlerRegistry } from '@/events';
|
|
@@ -153,6 +162,8 @@ export class Run<_T extends t.BaseGraphState> {
|
|
|
153
162
|
* lets callers assert the type they expect.
|
|
154
163
|
*/
|
|
155
164
|
private _interrupt: t.RunInterruptResult<unknown> | undefined;
|
|
165
|
+
/** Per-run sequence for batch-unique activity-label trace-seed fallbacks. */
|
|
166
|
+
private activityLabelSeq = 0;
|
|
156
167
|
private _haltedReason: string | undefined;
|
|
157
168
|
|
|
158
169
|
private constructor(config: Partial<t.RunConfig>) {
|
|
@@ -1478,6 +1489,291 @@ export class Run<_T extends t.BaseGraphState> {
|
|
|
1478
1489
|
await disposeLangfuseHandler(titleLangfuseHandler);
|
|
1479
1490
|
}
|
|
1480
1491
|
}
|
|
1492
|
+
|
|
1493
|
+
/**
|
|
1494
|
+
* Generates a short activity label for a completed tool/reasoning block
|
|
1495
|
+
* using a fast model. Mirrors `generateTitle`'s Langfuse wiring so the
|
|
1496
|
+
* call is traced under the conversation's session (sessionId from
|
|
1497
|
+
* `chainOptions.configurable.thread_id`) with its own tags — never as an
|
|
1498
|
+
* orphan trace. The payload contains no human messages by design: intent
|
|
1499
|
+
* comes from `lastAssistantText`, content from reasoning excerpts and
|
|
1500
|
+
* tool entries.
|
|
1501
|
+
*/
|
|
1502
|
+
async generateActivityLabel({
|
|
1503
|
+
provider,
|
|
1504
|
+
clientOptions,
|
|
1505
|
+
entries,
|
|
1506
|
+
thinkingExcerpts,
|
|
1507
|
+
lastAssistantText,
|
|
1508
|
+
prompt,
|
|
1509
|
+
charLimit = 600,
|
|
1510
|
+
chainOptions,
|
|
1511
|
+
traceSeed,
|
|
1512
|
+
agentId,
|
|
1513
|
+
}: t.RunActivityLabelOptions): Promise<{ label?: string }> {
|
|
1514
|
+
if (
|
|
1515
|
+
entries.length === 0 &&
|
|
1516
|
+
!(thinkingExcerpts && thinkingExcerpts.length > 0)
|
|
1517
|
+
) {
|
|
1518
|
+
return {};
|
|
1519
|
+
}
|
|
1520
|
+
const labelSeq = ++this.activityLabelSeq;
|
|
1521
|
+
|
|
1522
|
+
/** Resolve the LABELED agent's context: its Langfuse overlay carries the
|
|
1523
|
+
* trace metadata and the tool-output redaction policy that must govern
|
|
1524
|
+
* this label. */
|
|
1525
|
+
const requestedContext =
|
|
1526
|
+
this.Graph == null || agentId == null
|
|
1527
|
+
? undefined
|
|
1528
|
+
: this.Graph.agentContexts.get(agentId);
|
|
1529
|
+
/** Fail closed: an explicit but unknown/stale `agentId` must NOT silently
|
|
1530
|
+
* fall back to the default agent, whose redaction policy may be weaker
|
|
1531
|
+
* than the labeled agent's. Skip generation entirely instead. */
|
|
1532
|
+
if (agentId != null && requestedContext == null) {
|
|
1533
|
+
return {};
|
|
1534
|
+
}
|
|
1535
|
+
const labelContext =
|
|
1536
|
+
this.Graph == null
|
|
1537
|
+
? undefined
|
|
1538
|
+
: (requestedContext ??
|
|
1539
|
+
this.Graph.agentContexts.get(this.Graph.defaultAgentId));
|
|
1540
|
+
const traceMetadata = createLangfuseTraceMetadata({
|
|
1541
|
+
messageId: 'activity-label-' + this.id,
|
|
1542
|
+
agentName: labelContext?.name,
|
|
1543
|
+
});
|
|
1544
|
+
const labelRunName = getLangfuseTraceName(
|
|
1545
|
+
traceMetadata,
|
|
1546
|
+
'LibreChat Activity Label'
|
|
1547
|
+
);
|
|
1548
|
+
|
|
1549
|
+
/** Shallow-cloned: activity labels run once per tool batch, and writing
|
|
1550
|
+
* the Langfuse handler back onto a host-reused `chainOptions` would
|
|
1551
|
+
* accumulate duplicate callbacks across batches. */
|
|
1552
|
+
const labelChainOptions = {
|
|
1553
|
+
...(chainOptions ?? {}),
|
|
1554
|
+
} as Partial<RunnableConfig> & {
|
|
1555
|
+
configurable?: Record<string, unknown>;
|
|
1556
|
+
};
|
|
1557
|
+
const labelUserId =
|
|
1558
|
+
typeof labelChainOptions.configurable?.user_id === 'string'
|
|
1559
|
+
? (labelChainOptions.configurable.user_id as string)
|
|
1560
|
+
: undefined;
|
|
1561
|
+
const labelSessionId =
|
|
1562
|
+
typeof labelChainOptions.configurable?.thread_id === 'string'
|
|
1563
|
+
? (labelChainOptions.configurable.thread_id as string)
|
|
1564
|
+
: undefined;
|
|
1565
|
+
const labelLangfuseConfig = resolveLangfuseConfig(
|
|
1566
|
+
this.langfuse,
|
|
1567
|
+
labelContext?.langfuse
|
|
1568
|
+
);
|
|
1569
|
+
initializeLangfuseTracing(labelLangfuseConfig);
|
|
1570
|
+
/** Seed policy, threading two constraints:
|
|
1571
|
+
* 1. `runWithLangfuseRuntimeContext` SPREADS the surrounding context, so
|
|
1572
|
+
* an absent seed INHERITS the parent run's and collapses every label
|
|
1573
|
+
* into that trace. When a parent seed is active we must override it
|
|
1574
|
+
* with a per-label one.
|
|
1575
|
+
* 2. Without deterministic tracing there is no parent seed, and forcing
|
|
1576
|
+
* one here would make label trace ids deterministic when neither
|
|
1577
|
+
* `processStream` nor `generateTitle` are — so leave it unset.
|
|
1578
|
+
* Seeded when determinism is opted into OR a parent seed is live;
|
|
1579
|
+
* otherwise unseeded, matching the other generation paths. */
|
|
1580
|
+
const inheritedTraceSeed = getTraceIdSeed();
|
|
1581
|
+
const labelTraceSeed =
|
|
1582
|
+
labelLangfuseConfig?.deterministicTraceId === true ||
|
|
1583
|
+
inheritedTraceSeed != null
|
|
1584
|
+
? (traceSeed ?? `activity-label-${this.id}-${labelSeq}`)
|
|
1585
|
+
: undefined;
|
|
1586
|
+
const labelRuntimeScope = resolveLangfuseRuntimeScope({
|
|
1587
|
+
runLangfuse: this.langfuse,
|
|
1588
|
+
langfuseOverlay: labelContext?.langfuse,
|
|
1589
|
+
traceIdSeed: labelTraceSeed,
|
|
1590
|
+
});
|
|
1591
|
+
/** Handler only when a session id resolved from
|
|
1592
|
+
* `chainOptions.configurable.thread_id`: without it the label call has
|
|
1593
|
+
* no conversation identity, and tracing it would create an orphan
|
|
1594
|
+
* trace outside any session — worse than not tracing at all. */
|
|
1595
|
+
/** Declared then conditionally assigned (title precedent): a ternary
|
|
1596
|
+
* around the object literal makes eslint's indent rule and prettier
|
|
1597
|
+
* disagree, and both gate CI. */
|
|
1598
|
+
let labelLangfuseHandler: CallbackEntry | undefined;
|
|
1599
|
+
if (labelSessionId != null) {
|
|
1600
|
+
labelLangfuseHandler = createLangfuseHandler({
|
|
1601
|
+
langfuse: labelLangfuseConfig,
|
|
1602
|
+
userId: labelUserId,
|
|
1603
|
+
sessionId: labelSessionId,
|
|
1604
|
+
traceMetadata,
|
|
1605
|
+
tags: ['librechat', 'activity-label'],
|
|
1606
|
+
traceIdSeed:
|
|
1607
|
+
labelLangfuseConfig?.deterministicTraceId === true
|
|
1608
|
+
? labelTraceSeed
|
|
1609
|
+
: undefined,
|
|
1610
|
+
});
|
|
1611
|
+
}
|
|
1612
|
+
if (labelLangfuseHandler != null) {
|
|
1613
|
+
labelChainOptions.callbacks = appendCallbacks(
|
|
1614
|
+
labelChainOptions.callbacks,
|
|
1615
|
+
[labelLangfuseHandler]
|
|
1616
|
+
);
|
|
1617
|
+
}
|
|
1618
|
+
|
|
1619
|
+
/** The label prompt becomes Langfuse generation input, so the resolved
|
|
1620
|
+
* tool-output redaction policy (global disable / redactedToolNames)
|
|
1621
|
+
* applies to it exactly as to structured tool observations. */
|
|
1622
|
+
let redaction = hasToolOutputTracingConfig(
|
|
1623
|
+
this.langfuse,
|
|
1624
|
+
labelContext?.langfuse
|
|
1625
|
+
)
|
|
1626
|
+
? resolveToolOutputTracingConfig(this.langfuse, labelContext?.langfuse)
|
|
1627
|
+
: undefined;
|
|
1628
|
+
/** Multi-agent graph with no `agentId`: the caller did not say WHICH
|
|
1629
|
+
* agent ran this batch, so resolving from the default agent could trace
|
|
1630
|
+
* raw output that a stricter sibling's policy forbids. Fold every
|
|
1631
|
+
* agent's policy into the strictest one instead of guessing. */
|
|
1632
|
+
const agentContexts = this.Graph?.agentContexts;
|
|
1633
|
+
if (agentId == null && agentContexts != null && agentContexts.size > 1) {
|
|
1634
|
+
for (const context of agentContexts.values()) {
|
|
1635
|
+
if (!hasToolOutputTracingConfig(this.langfuse, context.langfuse)) {
|
|
1636
|
+
continue;
|
|
1637
|
+
}
|
|
1638
|
+
const candidate = resolveToolOutputTracingConfig(
|
|
1639
|
+
this.langfuse,
|
|
1640
|
+
context.langfuse
|
|
1641
|
+
);
|
|
1642
|
+
if (redaction == null) {
|
|
1643
|
+
redaction = candidate;
|
|
1644
|
+
continue;
|
|
1645
|
+
}
|
|
1646
|
+
redaction = {
|
|
1647
|
+
enabled: redaction.enabled === false ? false : candidate.enabled,
|
|
1648
|
+
redactedToolNames: new Set([
|
|
1649
|
+
...redaction.redactedToolNames,
|
|
1650
|
+
...candidate.redactedToolNames,
|
|
1651
|
+
]),
|
|
1652
|
+
redactedToolNameMatchMode:
|
|
1653
|
+
redaction.redactedToolNameMatchMode === 'partial' ||
|
|
1654
|
+
candidate.redactedToolNameMatchMode === 'partial'
|
|
1655
|
+
? 'partial'
|
|
1656
|
+
: 'exact',
|
|
1657
|
+
redactionText: redaction.redactionText,
|
|
1658
|
+
};
|
|
1659
|
+
}
|
|
1660
|
+
}
|
|
1661
|
+
/** An active redaction policy suppresses free-form reasoning/intent, so
|
|
1662
|
+
* a reasoning-only block has nothing describable left — skip the model
|
|
1663
|
+
* call rather than paying for a label built from the prompt alone. */
|
|
1664
|
+
const freeFormSuppressed =
|
|
1665
|
+
redaction != null &&
|
|
1666
|
+
(redaction.enabled === false || redaction.redactedToolNames.size > 0);
|
|
1667
|
+
if (entries.length === 0 && freeFormSuppressed) {
|
|
1668
|
+
return {};
|
|
1669
|
+
}
|
|
1670
|
+
const userPrompt = buildActivityLabelPrompt({
|
|
1671
|
+
entries,
|
|
1672
|
+
charLimit,
|
|
1673
|
+
thinkingExcerpts,
|
|
1674
|
+
lastAssistantText,
|
|
1675
|
+
redaction,
|
|
1676
|
+
});
|
|
1677
|
+
|
|
1678
|
+
const model = initializeModel({
|
|
1679
|
+
provider,
|
|
1680
|
+
clientOptions: {
|
|
1681
|
+
...(clientOptions ?? {}),
|
|
1682
|
+
streaming: false,
|
|
1683
|
+
} as t.ClientOptions,
|
|
1684
|
+
}) as t.ChatModelInstance;
|
|
1685
|
+
|
|
1686
|
+
/** Distinct run id per label call: callback/tracing integrations key
|
|
1687
|
+
* in-flight runs by it, so reusing the parent run's id would collide
|
|
1688
|
+
* across successive (or concurrent) label batches. */
|
|
1689
|
+
const labelRunId = `${this.id}-activity-${labelSeq}`;
|
|
1690
|
+
const invokeConfig = Object.assign({}, labelChainOptions, {
|
|
1691
|
+
run_id: labelRunId,
|
|
1692
|
+
runId: labelRunId,
|
|
1693
|
+
runName: labelChainOptions.runName ?? labelRunName,
|
|
1694
|
+
}) as Partial<RunnableConfig>;
|
|
1695
|
+
|
|
1696
|
+
const invokeLabel = (
|
|
1697
|
+
runtimeConfig: Partial<RunnableConfig>
|
|
1698
|
+
): Promise<unknown> =>
|
|
1699
|
+
withLangfuseAttributes(
|
|
1700
|
+
{
|
|
1701
|
+
langfuse: labelLangfuseConfig,
|
|
1702
|
+
userId: labelUserId,
|
|
1703
|
+
sessionId: labelSessionId,
|
|
1704
|
+
traceName: runtimeConfig.runName ?? labelRunName,
|
|
1705
|
+
traceMetadata,
|
|
1706
|
+
tags: ['librechat', 'activity-label'],
|
|
1707
|
+
},
|
|
1708
|
+
() =>
|
|
1709
|
+
model.invoke(
|
|
1710
|
+
[
|
|
1711
|
+
new SystemMessage(prompt ?? ACTIVITY_LABEL_PROMPT),
|
|
1712
|
+
new HumanMessage(userPrompt),
|
|
1713
|
+
],
|
|
1714
|
+
runtimeConfig
|
|
1715
|
+
)
|
|
1716
|
+
);
|
|
1717
|
+
|
|
1718
|
+
const extractLabel = (response: unknown): string => {
|
|
1719
|
+
const content = (response as { content?: unknown } | null)?.content;
|
|
1720
|
+
let text = '';
|
|
1721
|
+
if (typeof content === 'string') {
|
|
1722
|
+
text = content;
|
|
1723
|
+
} else if (Array.isArray(content)) {
|
|
1724
|
+
text = content
|
|
1725
|
+
.map((block) =>
|
|
1726
|
+
typeof block === 'string'
|
|
1727
|
+
? block
|
|
1728
|
+
: ((block as { text?: string }).text ?? '')
|
|
1729
|
+
)
|
|
1730
|
+
.join('');
|
|
1731
|
+
}
|
|
1732
|
+
return text.trim().replace(/^["']|["']$/g, '');
|
|
1733
|
+
};
|
|
1734
|
+
|
|
1735
|
+
try {
|
|
1736
|
+
let response: unknown;
|
|
1737
|
+
try {
|
|
1738
|
+
response = await withLangfuseRuntimeScope(labelRuntimeScope, () =>
|
|
1739
|
+
invokeLabel(invokeConfig)
|
|
1740
|
+
);
|
|
1741
|
+
} catch (error) {
|
|
1742
|
+
/** Retry ONLY recognized callback/tracer failures (the EventStream
|
|
1743
|
+
* tracer class of errors the stripped-callbacks fallback exists
|
|
1744
|
+
* for). Aborts and provider failures rethrow — retrying those
|
|
1745
|
+
* doubles traffic/cost and can restart cancelled requests. */
|
|
1746
|
+
const aborted =
|
|
1747
|
+
(labelChainOptions as { signal?: AbortSignal }).signal?.aborted ===
|
|
1748
|
+
true || (error as Error | null)?.name === 'AbortError';
|
|
1749
|
+
const callbackFailure = /callback|tracer|event.?stream/i.test(
|
|
1750
|
+
String(
|
|
1751
|
+
(error as Error | null)?.stack ??
|
|
1752
|
+
(error as Error | null)?.message ??
|
|
1753
|
+
''
|
|
1754
|
+
)
|
|
1755
|
+
);
|
|
1756
|
+
if (aborted || !callbackFailure) {
|
|
1757
|
+
throw error;
|
|
1758
|
+
}
|
|
1759
|
+
const langfuseHandler = findCallback(
|
|
1760
|
+
invokeConfig.callbacks,
|
|
1761
|
+
isLangfuseCallbackHandler
|
|
1762
|
+
);
|
|
1763
|
+
const { callbacks: _cb, ...rest } = invokeConfig;
|
|
1764
|
+
const safeConfig = Object.assign({}, rest, {
|
|
1765
|
+
callbacks: langfuseHandler ? [langfuseHandler] : [],
|
|
1766
|
+
});
|
|
1767
|
+
response = await withLangfuseRuntimeScope(labelRuntimeScope, () =>
|
|
1768
|
+
invokeLabel(safeConfig as Partial<RunnableConfig>)
|
|
1769
|
+
);
|
|
1770
|
+
}
|
|
1771
|
+
const label = extractLabel(response);
|
|
1772
|
+
return label.length > 0 ? { label } : {};
|
|
1773
|
+
} finally {
|
|
1774
|
+
await disposeLangfuseHandler(labelLangfuseHandler);
|
|
1775
|
+
}
|
|
1776
|
+
}
|
|
1481
1777
|
}
|
|
1482
1778
|
|
|
1483
1779
|
function findLastMessageOfType(
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import type { ActivityLabelToolEntry } from '@/types/activityLabel';
|
|
2
|
+
import { LANGFUSE_TOOL_OUTPUT_REDACTION_TEXT } from '@/langfuseToolOutputTracing';
|
|
3
|
+
import { buildActivityLabelPrompt } from '@/prompts/activityLabel';
|
|
4
|
+
import { resolveToolOutputTracingConfig } from '@/langfuseConfig';
|
|
5
|
+
|
|
6
|
+
const entries: ActivityLabelToolEntry[] = [
|
|
7
|
+
{
|
|
8
|
+
toolName: 'web_search',
|
|
9
|
+
toolInput: { query: 'runtime versions' },
|
|
10
|
+
toolOutput: 'PUBLIC_SEARCH_RESULTS',
|
|
11
|
+
status: 'success',
|
|
12
|
+
},
|
|
13
|
+
{
|
|
14
|
+
toolName: 'db_query',
|
|
15
|
+
toolInput: { sql: 'select 1' },
|
|
16
|
+
error: 'SECRET_CONNECTION_STRING_LEAK',
|
|
17
|
+
status: 'error',
|
|
18
|
+
},
|
|
19
|
+
];
|
|
20
|
+
|
|
21
|
+
describe('buildActivityLabelPrompt redaction', () => {
|
|
22
|
+
it('embeds raw outputs and errors when no redaction policy resolves', () => {
|
|
23
|
+
const prompt = buildActivityLabelPrompt({ entries, charLimit: 600 });
|
|
24
|
+
expect(prompt).toContain('PUBLIC_SEARCH_RESULTS');
|
|
25
|
+
expect(prompt).toContain('SECRET_CONNECTION_STRING_LEAK');
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
it('redacts every outcome when tool-output tracing is globally disabled', () => {
|
|
29
|
+
const redaction = resolveToolOutputTracingConfig({
|
|
30
|
+
toolOutputTracing: { enabled: false },
|
|
31
|
+
});
|
|
32
|
+
const prompt = buildActivityLabelPrompt({
|
|
33
|
+
entries,
|
|
34
|
+
charLimit: 600,
|
|
35
|
+
redaction,
|
|
36
|
+
});
|
|
37
|
+
expect(prompt).not.toContain('PUBLIC_SEARCH_RESULTS');
|
|
38
|
+
expect(prompt).not.toContain('SECRET_CONNECTION_STRING_LEAK');
|
|
39
|
+
expect(prompt).toContain(LANGFUSE_TOOL_OUTPUT_REDACTION_TEXT);
|
|
40
|
+
/** Tool names and inputs stay — matching the span processor, which
|
|
41
|
+
* redacts output fields only. */
|
|
42
|
+
expect(prompt).toContain('web_search');
|
|
43
|
+
expect(prompt).toContain('runtime versions');
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
it('drops reasoning excerpts when any batch entry is redacted', () => {
|
|
47
|
+
const redaction = resolveToolOutputTracingConfig({
|
|
48
|
+
toolOutputTracing: { redactedToolNames: ['db_query'] },
|
|
49
|
+
});
|
|
50
|
+
const prompt = buildActivityLabelPrompt({
|
|
51
|
+
entries,
|
|
52
|
+
charLimit: 600,
|
|
53
|
+
thinkingExcerpts: [
|
|
54
|
+
'The db_query returned SECRET_CONNECTION_STRING_LEAK earlier',
|
|
55
|
+
],
|
|
56
|
+
redaction,
|
|
57
|
+
});
|
|
58
|
+
expect(prompt).not.toContain('Reasoning excerpts');
|
|
59
|
+
expect(prompt).not.toContain('SECRET_CONNECTION_STRING_LEAK');
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
it('drops free-form context under ANY active policy, even with no matching entry', () => {
|
|
63
|
+
/** Reasoning/intent can quote output from an EARLIER call to the
|
|
64
|
+
* redacted tool that this batch does not contain, so an active policy
|
|
65
|
+
* suppresses free-form prose regardless of this batch's entries. */
|
|
66
|
+
const redaction = resolveToolOutputTracingConfig({
|
|
67
|
+
toolOutputTracing: { redactedToolNames: ['unrelated_tool'] },
|
|
68
|
+
});
|
|
69
|
+
const prompt = buildActivityLabelPrompt({
|
|
70
|
+
entries,
|
|
71
|
+
charLimit: 600,
|
|
72
|
+
thinkingExcerpts: ['Comparing versions across sources'],
|
|
73
|
+
lastAssistantText: 'Checking the unrelated_tool result from before',
|
|
74
|
+
redaction,
|
|
75
|
+
});
|
|
76
|
+
expect(prompt).not.toContain('Comparing versions across sources');
|
|
77
|
+
expect(prompt).not.toContain('Intent');
|
|
78
|
+
/** Non-matching entries keep their own outcomes. */
|
|
79
|
+
expect(prompt).toContain('PUBLIC_SEARCH_RESULTS');
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
it('keeps free-form context when no redaction policy is configured', () => {
|
|
83
|
+
const prompt = buildActivityLabelPrompt({
|
|
84
|
+
entries,
|
|
85
|
+
charLimit: 600,
|
|
86
|
+
thinkingExcerpts: ['Comparing versions across sources'],
|
|
87
|
+
lastAssistantText: 'Verifying each runtime',
|
|
88
|
+
redaction: undefined,
|
|
89
|
+
});
|
|
90
|
+
expect(prompt).toContain('Comparing versions across sources');
|
|
91
|
+
expect(prompt).toContain('Verifying each runtime');
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
it('bounds serialization of oversized structured tool output', () => {
|
|
95
|
+
const huge = Array.from({ length: 50_000 }, (_, i) => ({
|
|
96
|
+
id: i,
|
|
97
|
+
blob: 'x'.repeat(200),
|
|
98
|
+
}));
|
|
99
|
+
const prompt = buildActivityLabelPrompt({
|
|
100
|
+
entries: [
|
|
101
|
+
{
|
|
102
|
+
toolName: 'db_rows',
|
|
103
|
+
toolInput: { sql: 'select *' },
|
|
104
|
+
toolOutput: huge,
|
|
105
|
+
status: 'success',
|
|
106
|
+
},
|
|
107
|
+
],
|
|
108
|
+
charLimit: 600,
|
|
109
|
+
});
|
|
110
|
+
/** Degrades to a shape summary instead of materializing ~10MB of JSON. */
|
|
111
|
+
expect(prompt).toContain('[Array(50000)]');
|
|
112
|
+
expect(prompt.length).toBeLessThan(2000);
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
it('redacts only named tools, including their error text', () => {
|
|
116
|
+
const redaction = resolveToolOutputTracingConfig({
|
|
117
|
+
toolOutputTracing: { redactedToolNames: ['db_query'] },
|
|
118
|
+
});
|
|
119
|
+
const prompt = buildActivityLabelPrompt({
|
|
120
|
+
entries,
|
|
121
|
+
charLimit: 600,
|
|
122
|
+
redaction,
|
|
123
|
+
});
|
|
124
|
+
expect(prompt).toContain('PUBLIC_SEARCH_RESULTS');
|
|
125
|
+
expect(prompt).not.toContain('SECRET_CONNECTION_STRING_LEAK');
|
|
126
|
+
expect(prompt).toContain(LANGFUSE_TOOL_OUTPUT_REDACTION_TEXT);
|
|
127
|
+
});
|
|
128
|
+
});
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import {
|
|
2
|
+
resolveLangfuseRuntimeScope,
|
|
3
|
+
withLangfuseRuntimeScope,
|
|
4
|
+
} from '@/langfuseRuntimeScope';
|
|
5
|
+
import { getTraceIdSeed } from '@/langfuseRuntimeContext';
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* `generateActivityLabel` passes one label seed to both the Langfuse handler
|
|
9
|
+
* and the runtime scope it invokes under. This proves the mechanism: a label
|
|
10
|
+
* scope's seed must override an inherited (parent run) seed for the duration
|
|
11
|
+
* of the label call, then restore — otherwise per-batch label generations
|
|
12
|
+
* collapse into the main run trace under deterministic tracing.
|
|
13
|
+
*
|
|
14
|
+
* Asserted through the ALS runtime-context channel (`getTraceIdSeed`), which
|
|
15
|
+
* the trace id generator consults alongside OTel context; tests run without
|
|
16
|
+
* a registered OTel context manager, so the OTel channel is a no-op here.
|
|
17
|
+
*/
|
|
18
|
+
describe('activity-label trace seed scoping', () => {
|
|
19
|
+
it('overrides an inherited run seed for the nested scope only', () => {
|
|
20
|
+
const runScope = resolveLangfuseRuntimeScope({ traceIdSeed: 'run-seed' });
|
|
21
|
+
withLangfuseRuntimeScope(runScope, () => {
|
|
22
|
+
expect(getTraceIdSeed()).toBe('run-seed');
|
|
23
|
+
|
|
24
|
+
const labelScope = resolveLangfuseRuntimeScope({
|
|
25
|
+
traceIdSeed: 'run-1-activity-3',
|
|
26
|
+
});
|
|
27
|
+
withLangfuseRuntimeScope(labelScope, () => {
|
|
28
|
+
expect(getTraceIdSeed()).toBe('run-1-activity-3');
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
expect(getTraceIdSeed()).toBe('run-seed');
|
|
32
|
+
});
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
it('keeps distinct seeds for successive label scopes', () => {
|
|
36
|
+
const seeds: Array<string | undefined> = [];
|
|
37
|
+
for (const seed of ['run-1-activity-0', 'run-1-activity-1']) {
|
|
38
|
+
withLangfuseRuntimeScope(
|
|
39
|
+
resolveLangfuseRuntimeScope({ traceIdSeed: seed }),
|
|
40
|
+
() => {
|
|
41
|
+
seeds.push(getTraceIdSeed());
|
|
42
|
+
}
|
|
43
|
+
);
|
|
44
|
+
}
|
|
45
|
+
expect(seeds).toEqual(['run-1-activity-0', 'run-1-activity-1']);
|
|
46
|
+
});
|
|
47
|
+
});
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
// src/specs/bedrock-toolless.live.test.ts
|
|
2
|
+
/**
|
|
3
|
+
* Live Bedrock verification for the tool-less-destination toolConfig fix.
|
|
4
|
+
*
|
|
5
|
+
* A tool-less agent in a multi-agent graph inherits the prior agent's
|
|
6
|
+
* toolUse/toolResult history. Because it binds no tools, Bedrock's Converse API
|
|
7
|
+
* rejects the request ("The toolConfig field must be defined when using toolUse
|
|
8
|
+
* and toolResult content blocks"). `foldToolBlocksForToollessAgent` folds that
|
|
9
|
+
* history into text so the request becomes valid.
|
|
10
|
+
*
|
|
11
|
+
* Run with:
|
|
12
|
+
* RUN_BEDROCK_LIVE_TESTS=1 BEDROCK_AWS_ACCESS_KEY_ID=... BEDROCK_AWS_SECRET_ACCESS_KEY=... \
|
|
13
|
+
* BEDROCK_AWS_DEFAULT_REGION=us-west-2 npm test -- bedrock-toolless.live.test.ts --runInBand
|
|
14
|
+
*/
|
|
15
|
+
import { config as dotenvConfig } from 'dotenv';
|
|
16
|
+
dotenvConfig();
|
|
17
|
+
|
|
18
|
+
import {
|
|
19
|
+
AIMessage,
|
|
20
|
+
HumanMessage,
|
|
21
|
+
ToolMessage,
|
|
22
|
+
} from '@langchain/core/messages';
|
|
23
|
+
import { beforeAll, describe, expect, it } from '@jest/globals';
|
|
24
|
+
import type { BaseMessage } from '@langchain/core/messages';
|
|
25
|
+
import type * as t from '@/types';
|
|
26
|
+
import { Providers } from '@/common';
|
|
27
|
+
import { initializeModel } from '@/llm/init';
|
|
28
|
+
import { foldToolBlocksForToollessAgent } from '@/messages';
|
|
29
|
+
|
|
30
|
+
const accessKeyId =
|
|
31
|
+
process.env.BEDROCK_AWS_ACCESS_KEY_ID ?? process.env.AWS_ACCESS_KEY_ID;
|
|
32
|
+
const secretAccessKey =
|
|
33
|
+
process.env.BEDROCK_AWS_SECRET_ACCESS_KEY ??
|
|
34
|
+
process.env.AWS_SECRET_ACCESS_KEY;
|
|
35
|
+
|
|
36
|
+
const shouldRunLive =
|
|
37
|
+
process.env.RUN_BEDROCK_LIVE_TESTS === '1' &&
|
|
38
|
+
accessKeyId != null &&
|
|
39
|
+
accessKeyId !== '' &&
|
|
40
|
+
secretAccessKey != null &&
|
|
41
|
+
secretAccessKey !== '';
|
|
42
|
+
|
|
43
|
+
const describeIfLive = shouldRunLive ? describe : describe.skip;
|
|
44
|
+
|
|
45
|
+
const MODEL =
|
|
46
|
+
process.env.LIVE_BEDROCK_MODEL ??
|
|
47
|
+
'us.anthropic.claude-sonnet-4-5-20250929-v1:0';
|
|
48
|
+
const REGION =
|
|
49
|
+
process.env.BEDROCK_AWS_DEFAULT_REGION ??
|
|
50
|
+
process.env.AWS_REGION ??
|
|
51
|
+
'us-west-2';
|
|
52
|
+
|
|
53
|
+
/** History a tool-less destination inherits: a completed tool call, then a
|
|
54
|
+
* follow-up user turn that itself invokes no tool. */
|
|
55
|
+
function toollessHistory(): BaseMessage[] {
|
|
56
|
+
return [
|
|
57
|
+
new HumanMessage('Search my files for the roadmap.'),
|
|
58
|
+
new AIMessage({
|
|
59
|
+
content: '',
|
|
60
|
+
tool_calls: [
|
|
61
|
+
{
|
|
62
|
+
id: 'tt_live_1',
|
|
63
|
+
name: 'file_search',
|
|
64
|
+
args: { query: 'roadmap' },
|
|
65
|
+
type: 'tool_call',
|
|
66
|
+
},
|
|
67
|
+
],
|
|
68
|
+
}),
|
|
69
|
+
new ToolMessage({
|
|
70
|
+
content: 'Found: roadmap.md — Q3 goals and milestones.',
|
|
71
|
+
tool_call_id: 'tt_live_1',
|
|
72
|
+
name: 'file_search',
|
|
73
|
+
}),
|
|
74
|
+
new AIMessage('I found roadmap.md with your Q3 goals.'),
|
|
75
|
+
new HumanMessage('thanks!'),
|
|
76
|
+
];
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
describeIfLive('Bedrock tool-less destination (live)', () => {
|
|
80
|
+
let clientOptions: t.BedrockConverseClientOptions;
|
|
81
|
+
|
|
82
|
+
beforeAll(() => {
|
|
83
|
+
// Force SigV4 with the explicit keys; the Bedrock API-key (bearer) auth
|
|
84
|
+
// scheme otherwise takes precedence in the AWS SDK.
|
|
85
|
+
delete process.env.AWS_BEARER_TOKEN_BEDROCK;
|
|
86
|
+
// `shouldRunLive` already guarantees both keys are set; `?? ''` only keeps
|
|
87
|
+
// the credential fields typed as strings.
|
|
88
|
+
clientOptions = {
|
|
89
|
+
region: REGION,
|
|
90
|
+
model: MODEL,
|
|
91
|
+
credentials: {
|
|
92
|
+
accessKeyId: accessKeyId ?? '',
|
|
93
|
+
secretAccessKey: secretAccessKey ?? '',
|
|
94
|
+
},
|
|
95
|
+
};
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
it('rejects inherited tool history when the agent binds no tools', async () => {
|
|
99
|
+
const model = initializeModel({
|
|
100
|
+
provider: Providers.BEDROCK,
|
|
101
|
+
clientOptions,
|
|
102
|
+
tools: undefined,
|
|
103
|
+
});
|
|
104
|
+
await expect(model.invoke(toollessHistory())).rejects.toThrow(
|
|
105
|
+
/toolConfig field must be defined/i
|
|
106
|
+
);
|
|
107
|
+
}, 30000);
|
|
108
|
+
|
|
109
|
+
it('succeeds once inherited tool blocks are folded to text', async () => {
|
|
110
|
+
const model = initializeModel({
|
|
111
|
+
provider: Providers.BEDROCK,
|
|
112
|
+
clientOptions,
|
|
113
|
+
tools: undefined,
|
|
114
|
+
});
|
|
115
|
+
const folded = foldToolBlocksForToollessAgent(toollessHistory());
|
|
116
|
+
const res = await model.invoke(folded);
|
|
117
|
+
const text =
|
|
118
|
+
typeof res.content === 'string'
|
|
119
|
+
? res.content
|
|
120
|
+
: JSON.stringify(res.content);
|
|
121
|
+
expect(text.length).toBeGreaterThan(0);
|
|
122
|
+
}, 30000);
|
|
123
|
+
});
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import type { RunnableConfig } from '@langchain/core/runnables';
|
|
2
|
+
import type { ClientOptions } from '@/types/llm';
|
|
3
|
+
import type { Providers } from '@/common';
|
|
4
|
+
|
|
5
|
+
/** One tool call's contribution to the label payload (host-assembled). */
|
|
6
|
+
export type ActivityLabelToolEntry = {
|
|
7
|
+
toolName: string;
|
|
8
|
+
toolInput: unknown;
|
|
9
|
+
toolOutput?: unknown;
|
|
10
|
+
error?: string;
|
|
11
|
+
status: 'success' | 'error';
|
|
12
|
+
};
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Options for `Run.generateActivityLabel`. The payload deliberately contains
|
|
16
|
+
* NO human messages: intent context comes from the assistant's own last text
|
|
17
|
+
* (Claude Code's pattern) and the block's reasoning excerpts (claude.ai's
|
|
18
|
+
* pattern) — user text stays out of this low-scrutiny pathway entirely.
|
|
19
|
+
*
|
|
20
|
+
* This SDK defines NO activity-label graph event and never dispatches one.
|
|
21
|
+
* Label lifecycle streaming is entirely host-owned: a host claims its own
|
|
22
|
+
* content slots and emits on its own transport, with a payload shape only
|
|
23
|
+
* it defines. The SDK surface here is exactly this method plus the
|
|
24
|
+
* `activity_label` content type's formatter exclusions.
|
|
25
|
+
*/
|
|
26
|
+
export type RunActivityLabelOptions = {
|
|
27
|
+
provider: Providers;
|
|
28
|
+
clientOptions?: ClientOptions;
|
|
29
|
+
/**
|
|
30
|
+
* Agent that executed the labeled batch. Selects that agent's Langfuse
|
|
31
|
+
* overlay (trace metadata AND tool-output redaction policy) instead of
|
|
32
|
+
* the graph default — a stricter per-agent policy must not be bypassed
|
|
33
|
+
* by labeling work the default agent never performed.
|
|
34
|
+
*/
|
|
35
|
+
agentId?: string;
|
|
36
|
+
entries: ActivityLabelToolEntry[];
|
|
37
|
+
/** Truncated reasoning excerpts from the block being labeled. */
|
|
38
|
+
thinkingExcerpts?: string[];
|
|
39
|
+
/** Assistant's last text before the block (~200 chars), as intent context. */
|
|
40
|
+
lastAssistantText?: string;
|
|
41
|
+
/** Override for the default label system prompt. */
|
|
42
|
+
prompt?: string;
|
|
43
|
+
/** Per-entry serialization cap for the prompt. Default 600. */
|
|
44
|
+
charLimit?: number;
|
|
45
|
+
/** LangChain runnable config carrier (signal, callbacks, thread/user ids). */
|
|
46
|
+
chainOptions?: Partial<RunnableConfig> & {
|
|
47
|
+
configurable?: Record<string, unknown>;
|
|
48
|
+
};
|
|
49
|
+
/**
|
|
50
|
+
* Seed for deterministic Langfuse trace ids (e.g. `${runId}-${slotIndex}`)
|
|
51
|
+
* so each batch's label gets a distinct, reproducible trace. When omitted,
|
|
52
|
+
* a per-run sequence keeps batches from collapsing into one trace.
|
|
53
|
+
*/
|
|
54
|
+
traceSeed?: string;
|
|
55
|
+
};
|