@librechat/agents 3.3.8 → 3.3.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/graphs/Graph.cjs +47 -13
- package/dist/cjs/graphs/Graph.cjs.map +1 -1
- package/dist/cjs/graphs/MultiAgentGraph.cjs +35 -2
- package/dist/cjs/graphs/MultiAgentGraph.cjs.map +1 -1
- package/dist/cjs/hitl/askUserQuestion.cjs +3 -2
- package/dist/cjs/hitl/askUserQuestion.cjs.map +1 -1
- package/dist/cjs/instrumentation.cjs +18 -48
- package/dist/cjs/instrumentation.cjs.map +1 -1
- package/dist/cjs/langfuse.cjs +174 -29
- package/dist/cjs/langfuse.cjs.map +1 -1
- package/dist/cjs/langfuseConfig.cjs +12 -0
- package/dist/cjs/langfuseConfig.cjs.map +1 -1
- package/dist/cjs/langfuseRuntimeContext.cjs +23 -2
- package/dist/cjs/langfuseRuntimeContext.cjs.map +1 -1
- package/dist/cjs/langfuseRuntimeScope.cjs +39 -8
- package/dist/cjs/langfuseRuntimeScope.cjs.map +1 -1
- package/dist/cjs/langfuseSpanRegistry.cjs +95 -0
- package/dist/cjs/langfuseSpanRegistry.cjs.map +1 -0
- package/dist/cjs/langfuseTraceShaping.cjs +121 -4
- package/dist/cjs/langfuseTraceShaping.cjs.map +1 -1
- package/dist/cjs/llm/anthropic/utils/message_inputs.cjs +39 -15
- package/dist/cjs/llm/anthropic/utils/message_inputs.cjs.map +1 -1
- package/dist/cjs/llm/bedrock/utils/message_inputs.cjs +25 -5
- package/dist/cjs/llm/bedrock/utils/message_inputs.cjs.map +1 -1
- package/dist/cjs/llm/init.cjs +3 -3
- package/dist/cjs/llm/invoke.cjs +5 -5
- package/dist/cjs/llm/openai/index.cjs +1 -1
- package/dist/cjs/main.cjs +10 -10
- package/dist/cjs/messages/prune.cjs +13 -1
- package/dist/cjs/messages/prune.cjs.map +1 -1
- package/dist/cjs/prompts/activityLabel.cjs +24 -12
- package/dist/cjs/prompts/activityLabel.cjs.map +1 -1
- package/dist/cjs/run.cjs +47 -22
- package/dist/cjs/run.cjs.map +1 -1
- package/dist/cjs/session/messageSerialization.cjs +6 -0
- package/dist/cjs/session/messageSerialization.cjs.map +1 -1
- package/dist/cjs/stream.cjs +21 -10
- package/dist/cjs/stream.cjs.map +1 -1
- package/dist/cjs/summarization/node.cjs +5 -0
- package/dist/cjs/summarization/node.cjs.map +1 -1
- package/dist/cjs/tools/ToolNode.cjs +253 -24
- package/dist/cjs/tools/ToolNode.cjs.map +1 -1
- package/dist/cjs/tools/handlers.cjs +1 -1
- package/dist/cjs/tools/search/tool.cjs +1 -1
- package/dist/cjs/tools/subagent/SubagentExecutor.cjs +1 -1
- package/dist/cjs/utils/index.cjs +2 -2
- package/dist/esm/graphs/Graph.mjs +48 -14
- package/dist/esm/graphs/Graph.mjs.map +1 -1
- package/dist/esm/graphs/MultiAgentGraph.mjs +35 -2
- package/dist/esm/graphs/MultiAgentGraph.mjs.map +1 -1
- package/dist/esm/hitl/askUserQuestion.mjs +3 -2
- package/dist/esm/hitl/askUserQuestion.mjs.map +1 -1
- package/dist/esm/instrumentation.mjs +18 -48
- package/dist/esm/instrumentation.mjs.map +1 -1
- package/dist/esm/langfuse.mjs +176 -28
- package/dist/esm/langfuse.mjs.map +1 -1
- package/dist/esm/langfuseConfig.mjs +10 -1
- package/dist/esm/langfuseConfig.mjs.map +1 -1
- package/dist/esm/langfuseRuntimeContext.mjs +21 -3
- package/dist/esm/langfuseRuntimeContext.mjs.map +1 -1
- package/dist/esm/langfuseRuntimeScope.mjs +39 -10
- package/dist/esm/langfuseRuntimeScope.mjs.map +1 -1
- package/dist/esm/langfuseSpanRegistry.mjs +91 -0
- package/dist/esm/langfuseSpanRegistry.mjs.map +1 -0
- package/dist/esm/langfuseTraceShaping.mjs +121 -4
- package/dist/esm/langfuseTraceShaping.mjs.map +1 -1
- package/dist/esm/llm/anthropic/utils/message_inputs.mjs +39 -15
- package/dist/esm/llm/anthropic/utils/message_inputs.mjs.map +1 -1
- package/dist/esm/llm/bedrock/utils/message_inputs.mjs +25 -5
- package/dist/esm/llm/bedrock/utils/message_inputs.mjs.map +1 -1
- package/dist/esm/llm/init.mjs +2 -2
- package/dist/esm/llm/invoke.mjs +5 -5
- package/dist/esm/llm/openai/index.mjs +1 -1
- package/dist/esm/main.mjs +8 -8
- package/dist/esm/messages/prune.mjs +13 -1
- package/dist/esm/messages/prune.mjs.map +1 -1
- package/dist/esm/prompts/activityLabel.mjs +24 -12
- package/dist/esm/prompts/activityLabel.mjs.map +1 -1
- package/dist/esm/run.mjs +47 -22
- package/dist/esm/run.mjs.map +1 -1
- package/dist/esm/session/messageSerialization.mjs +6 -0
- package/dist/esm/session/messageSerialization.mjs.map +1 -1
- package/dist/esm/stream.mjs +21 -10
- package/dist/esm/stream.mjs.map +1 -1
- package/dist/esm/summarization/node.mjs +5 -0
- package/dist/esm/summarization/node.mjs.map +1 -1
- package/dist/esm/tools/ToolNode.mjs +254 -25
- package/dist/esm/tools/ToolNode.mjs.map +1 -1
- package/dist/esm/tools/handlers.mjs +1 -1
- package/dist/esm/tools/search/tool.mjs +1 -1
- package/dist/esm/tools/subagent/SubagentExecutor.mjs +1 -1
- package/dist/esm/utils/index.mjs +2 -2
- package/dist/types/graphs/Graph.d.ts +19 -0
- package/dist/types/hitl/askUserQuestion.d.ts +11 -1
- package/dist/types/langfuse.d.ts +16 -8
- package/dist/types/langfuseConfig.d.ts +6 -0
- package/dist/types/langfuseRuntimeContext.d.ts +27 -1
- package/dist/types/langfuseRuntimeScope.d.ts +17 -2
- package/dist/types/langfuseSpanRegistry.d.ts +17 -0
- package/dist/types/langfuseTraceShaping.d.ts +2 -1
- package/dist/types/llm/anthropic/utils/message_inputs.d.ts +1 -0
- package/dist/types/session/types.d.ts +1 -0
- package/dist/types/tools/ToolNode.d.ts +7 -1
- package/dist/types/types/hitl.d.ts +8 -0
- package/dist/types/types/tools.d.ts +30 -0
- package/package.json +7 -4
- package/src/__tests__/stream.eagerArgsDivergence.test.ts +753 -0
- package/src/graphs/Graph.ts +69 -20
- package/src/graphs/MultiAgentGraph.ts +56 -2
- package/src/graphs/__tests__/composition.smoke.test.ts +4 -0
- package/src/hitl/askUserQuestion.ts +14 -1
- package/src/instrumentation.ts +35 -77
- package/src/langfuse.ts +320 -43
- package/src/langfuseConfig.ts +24 -0
- package/src/langfuseRuntimeContext.ts +43 -1
- package/src/langfuseRuntimeScope.ts +94 -21
- package/src/langfuseSpanRegistry.ts +131 -0
- package/src/langfuseTraceShaping.ts +194 -7
- package/src/llm/anthropic/utils/message_inputs.ts +70 -19
- package/src/llm/anthropic/utils/streaming-tool-input.test.ts +186 -11
- package/src/llm/bedrock/utils/message_inputs.test.ts +120 -4
- package/src/llm/bedrock/utils/message_inputs.ts +32 -7
- package/src/messages/prune.ts +12 -1
- package/src/prompts/activityLabel.ts +23 -6
- package/src/run.ts +76 -45
- package/src/scripts/activity-labels/captured.json +56 -0
- package/src/scripts/activity-labels/checks.cjs +205 -0
- package/src/scripts/activity-labels/corpus.cjs +473 -0
- package/src/scripts/activity-labels/report.cjs +203 -0
- package/src/scripts/activity-labels/rescore.cjs +102 -0
- package/src/scripts/activity-labels/run.ts +705 -0
- package/src/scripts/activity-labels/variants.ts +71 -0
- package/src/session/messageSerialization.ts +12 -1
- package/src/session/types.ts +1 -0
- package/src/specs/activity-label-prompt.test.ts +26 -10
- package/src/specs/agent-handoffs.test.ts +306 -0
- package/src/specs/langfuse-callbacks.test.ts +456 -0
- package/src/specs/langfuse-routing.integration.test.ts +138 -1
- package/src/specs/langfuse-span-registry.test.ts +70 -0
- package/src/specs/langfuse-trace-shaping.test.ts +294 -0
- package/src/specs/prune.test.ts +38 -1
- package/src/stream.ts +70 -6
- package/src/summarization/node.ts +5 -0
- package/src/tools/ToolNode.ts +400 -9
- package/src/tools/__tests__/ToolNode.invalidToolCalls.test.ts +757 -0
- package/src/tools/__tests__/hitl.test.ts +58 -0
- package/src/types/hitl.ts +8 -0
- package/src/types/tools.ts +35 -1
package/src/run.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
// src/run.ts
|
|
2
|
+
import { nanoid } from 'nanoid';
|
|
2
3
|
import { PromptTemplate } from '@langchain/core/prompts';
|
|
3
4
|
import { RunnableLambda } from '@langchain/core/runnables';
|
|
4
5
|
import { AzureChatOpenAI, ChatOpenAI } from '@langchain/openai';
|
|
@@ -39,6 +40,12 @@ import {
|
|
|
39
40
|
ACTIVITY_LABEL_PROMPT,
|
|
40
41
|
buildActivityLabelPrompt,
|
|
41
42
|
} from '@/prompts/activityLabel';
|
|
43
|
+
import {
|
|
44
|
+
Callback,
|
|
45
|
+
GraphEvents,
|
|
46
|
+
TitleMethod,
|
|
47
|
+
DEFAULT_RECURSION_LIMIT,
|
|
48
|
+
} from '@/common';
|
|
42
49
|
import {
|
|
43
50
|
appendCallbacks,
|
|
44
51
|
findCallback,
|
|
@@ -49,16 +56,10 @@ import {
|
|
|
49
56
|
createTitleRunnable,
|
|
50
57
|
} from '@/utils/title';
|
|
51
58
|
import { createTokenCounter, encodingForModel } from '@/utils/tokens';
|
|
52
|
-
import { resolveMaxSeals } from '@/llm/preempt';
|
|
53
59
|
import { initializeLangfuseTracing } from './instrumentation';
|
|
54
|
-
import {
|
|
55
|
-
Callback,
|
|
56
|
-
GraphEvents,
|
|
57
|
-
TitleMethod,
|
|
58
|
-
DEFAULT_RECURSION_LIMIT,
|
|
59
|
-
} from '@/common';
|
|
60
60
|
import { MultiAgentGraph } from '@/graphs/MultiAgentGraph';
|
|
61
61
|
import { getTraceIdSeed } from '@/langfuseRuntimeContext';
|
|
62
|
+
import { resolveMaxSeals } from '@/llm/preempt';
|
|
62
63
|
import { StandardGraph } from '@/graphs/Graph';
|
|
63
64
|
import { initializeModel } from '@/llm/init';
|
|
64
65
|
import { HandlerRegistry } from '@/events';
|
|
@@ -828,6 +829,18 @@ export class Run<_T extends t.BaseGraphState> {
|
|
|
828
829
|
const traceName = config.runName ?? getLangfuseTraceName(traceMetadata);
|
|
829
830
|
const streamLangfuseConfig = this.getStreamLangfuseConfig(graph);
|
|
830
831
|
initializeLangfuseTracing(streamLangfuseConfig);
|
|
832
|
+
const streamRuntimeScope = resolveLangfuseRuntimeScope({
|
|
833
|
+
runLangfuse: streamLangfuseConfig,
|
|
834
|
+
langfuseOverlay: this.getStreamToolOutputTracingLangfuseConfig(graph),
|
|
835
|
+
traceIdSeed:
|
|
836
|
+
streamLangfuseConfig?.deterministicTraceId === true
|
|
837
|
+
? this.id
|
|
838
|
+
: undefined,
|
|
839
|
+
// The graph's per-execution stamp, NOT the public run id: public ids
|
|
840
|
+
// may repeat across concurrent executions (retries, tenant-local
|
|
841
|
+
// message ids), and equal stamps defeat foreign-scope rejection.
|
|
842
|
+
runId: graph.langfuseScopeRunId,
|
|
843
|
+
});
|
|
831
844
|
const langfuseHandler = createLangfuseHandler({
|
|
832
845
|
langfuse: streamLangfuseConfig,
|
|
833
846
|
userId,
|
|
@@ -838,6 +851,12 @@ export class Run<_T extends t.BaseGraphState> {
|
|
|
838
851
|
streamLangfuseConfig?.deterministicTraceId === true
|
|
839
852
|
? this.id
|
|
840
853
|
: undefined,
|
|
854
|
+
runId: graph.langfuseScopeRunId,
|
|
855
|
+
// The aggregate multi-agent policy from the runtime scope — the
|
|
856
|
+
// handler must restore THIS (not the primary agent's config-derived
|
|
857
|
+
// policy) when rejecting a foreign scope.
|
|
858
|
+
toolOutputTracing: streamRuntimeScope.toolOutputTracing,
|
|
859
|
+
traceName,
|
|
841
860
|
});
|
|
842
861
|
if (langfuseHandler != null) {
|
|
843
862
|
config.runName = traceName;
|
|
@@ -1057,27 +1076,18 @@ export class Run<_T extends t.BaseGraphState> {
|
|
|
1057
1076
|
// When opted in, seed the root trace id from this run's id so feedback /
|
|
1058
1077
|
// other external signals can be attached to the trace later without a
|
|
1059
1078
|
// lookup (see SeededTraceIdGenerator in ./instrumentation).
|
|
1060
|
-
await withLangfuseRuntimeScope(
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
langfuse: streamLangfuseConfig,
|
|
1073
|
-
userId,
|
|
1074
|
-
sessionId,
|
|
1075
|
-
traceName,
|
|
1076
|
-
traceMetadata,
|
|
1077
|
-
tags: ['librechat', 'agent'],
|
|
1078
|
-
},
|
|
1079
|
-
consumeStream
|
|
1080
|
-
)
|
|
1079
|
+
await withLangfuseRuntimeScope(streamRuntimeScope, () =>
|
|
1080
|
+
withLangfuseAttributes(
|
|
1081
|
+
{
|
|
1082
|
+
langfuse: streamLangfuseConfig,
|
|
1083
|
+
userId,
|
|
1084
|
+
sessionId,
|
|
1085
|
+
traceName,
|
|
1086
|
+
traceMetadata,
|
|
1087
|
+
tags: ['librechat', 'agent'],
|
|
1088
|
+
},
|
|
1089
|
+
consumeStream
|
|
1090
|
+
)
|
|
1081
1091
|
);
|
|
1082
1092
|
} catch (err) {
|
|
1083
1093
|
streamThrew = true;
|
|
@@ -1438,18 +1448,42 @@ export class Run<_T extends t.BaseGraphState> {
|
|
|
1438
1448
|
titlePromptTemplate,
|
|
1439
1449
|
}: t.RunTitleOptions): Promise<{ language?: string; title?: string }> {
|
|
1440
1450
|
let titleLangfuseHandler: CallbackEntry | undefined;
|
|
1441
|
-
let titleLangfuseConfig: t.LangfuseConfig | undefined;
|
|
1442
1451
|
let titleUserId: string | undefined;
|
|
1443
1452
|
let titleSessionId: string | undefined;
|
|
1444
1453
|
const titleContext =
|
|
1445
1454
|
this.Graph == null
|
|
1446
1455
|
? undefined
|
|
1447
1456
|
: this.Graph.agentContexts.get(this.Graph.defaultAgentId);
|
|
1457
|
+
const titleLangfuseConfig = resolveLangfuseConfig(
|
|
1458
|
+
this.langfuse,
|
|
1459
|
+
titleContext?.langfuse
|
|
1460
|
+
);
|
|
1448
1461
|
const traceMetadata = createLangfuseTraceMetadata({
|
|
1449
1462
|
messageId: 'title-' + this.id,
|
|
1450
1463
|
agentName: titleContext?.name,
|
|
1451
1464
|
});
|
|
1452
1465
|
const titleRunName = getLangfuseTraceName(traceMetadata, 'LibreChat Title');
|
|
1466
|
+
/** Scope identity carries an opaque per-execution component: public run
|
|
1467
|
+
* ids are unrestricted, so a purely derived id (`title-<runId>`) could
|
|
1468
|
+
* collide with an ordinary concurrent run literally named that way and
|
|
1469
|
+
* defeat foreign-scope rejection. */
|
|
1470
|
+
const titleScopeRunId = `title:${this.id}:${nanoid()}`;
|
|
1471
|
+
/** Seed policy mirrors `generateActivityLabel`:
|
|
1472
|
+
* `runWithLangfuseRuntimeContext` SPREADS the surrounding context, so an
|
|
1473
|
+
* absent seed INHERITS an active parent run's and collapses the title
|
|
1474
|
+
* into that run's trace. Seeded when determinism is opted into OR a
|
|
1475
|
+
* parent seed is live; otherwise unseeded, matching the other paths. */
|
|
1476
|
+
const inheritedTraceSeed = getTraceIdSeed();
|
|
1477
|
+
const titleRuntimeScope = resolveLangfuseRuntimeScope({
|
|
1478
|
+
runLangfuse: this.langfuse,
|
|
1479
|
+
langfuseOverlay: titleContext?.langfuse,
|
|
1480
|
+
traceIdSeed:
|
|
1481
|
+
titleLangfuseConfig?.deterministicTraceId === true ||
|
|
1482
|
+
inheritedTraceSeed != null
|
|
1483
|
+
? 'title-' + this.id
|
|
1484
|
+
: undefined,
|
|
1485
|
+
runId: titleScopeRunId,
|
|
1486
|
+
});
|
|
1453
1487
|
|
|
1454
1488
|
if (chainOptions != null) {
|
|
1455
1489
|
titleUserId =
|
|
@@ -1460,10 +1494,6 @@ export class Run<_T extends t.BaseGraphState> {
|
|
|
1460
1494
|
typeof chainOptions.configurable?.thread_id === 'string'
|
|
1461
1495
|
? chainOptions.configurable.thread_id
|
|
1462
1496
|
: undefined;
|
|
1463
|
-
titleLangfuseConfig = resolveLangfuseConfig(
|
|
1464
|
-
this.langfuse,
|
|
1465
|
-
titleContext?.langfuse
|
|
1466
|
-
);
|
|
1467
1497
|
initializeLangfuseTracing(titleLangfuseConfig);
|
|
1468
1498
|
titleLangfuseHandler = createLangfuseHandler({
|
|
1469
1499
|
langfuse: titleLangfuseConfig,
|
|
@@ -1475,6 +1505,9 @@ export class Run<_T extends t.BaseGraphState> {
|
|
|
1475
1505
|
titleLangfuseConfig?.deterministicTraceId === true
|
|
1476
1506
|
? 'title-' + this.id
|
|
1477
1507
|
: undefined,
|
|
1508
|
+
runId: titleScopeRunId,
|
|
1509
|
+
toolOutputTracing: titleRuntimeScope.toolOutputTracing,
|
|
1510
|
+
traceName: chainOptions.runName ?? titleRunName,
|
|
1478
1511
|
});
|
|
1479
1512
|
|
|
1480
1513
|
if (titleLangfuseHandler != null) {
|
|
@@ -1567,12 +1600,8 @@ export class Run<_T extends t.BaseGraphState> {
|
|
|
1567
1600
|
|
|
1568
1601
|
try {
|
|
1569
1602
|
try {
|
|
1570
|
-
return await withLangfuseRuntimeScope(
|
|
1571
|
-
|
|
1572
|
-
runLangfuse: this.langfuse,
|
|
1573
|
-
langfuseOverlay: titleContext?.langfuse,
|
|
1574
|
-
}),
|
|
1575
|
-
() => invokeTitleChain(invokeConfig)
|
|
1603
|
+
return await withLangfuseRuntimeScope(titleRuntimeScope, () =>
|
|
1604
|
+
invokeTitleChain(invokeConfig)
|
|
1576
1605
|
);
|
|
1577
1606
|
} catch (_e) {
|
|
1578
1607
|
// Fallback: strip callbacks to avoid EventStream tracer errors in certain environments
|
|
@@ -1585,12 +1614,8 @@ export class Run<_T extends t.BaseGraphState> {
|
|
|
1585
1614
|
const safeConfig = Object.assign({}, rest, {
|
|
1586
1615
|
callbacks: langfuseHandler ? [langfuseHandler] : [],
|
|
1587
1616
|
});
|
|
1588
|
-
return await withLangfuseRuntimeScope(
|
|
1589
|
-
|
|
1590
|
-
runLangfuse: this.langfuse,
|
|
1591
|
-
langfuseOverlay: titleContext?.langfuse,
|
|
1592
|
-
}),
|
|
1593
|
-
() => invokeTitleChain(safeConfig as Partial<RunnableConfig>)
|
|
1617
|
+
return await withLangfuseRuntimeScope(titleRuntimeScope, () =>
|
|
1618
|
+
invokeTitleChain(safeConfig as Partial<RunnableConfig>)
|
|
1594
1619
|
);
|
|
1595
1620
|
}
|
|
1596
1621
|
} finally {
|
|
@@ -1692,10 +1717,13 @@ export class Run<_T extends t.BaseGraphState> {
|
|
|
1692
1717
|
inheritedTraceSeed != null
|
|
1693
1718
|
? (traceSeed ?? `activity-label-${this.id}-${labelSeq}`)
|
|
1694
1719
|
: undefined;
|
|
1720
|
+
/** Opaque per-execution component: see `titleScopeRunId`. */
|
|
1721
|
+
const labelScopeRunId = `activity-label:${this.id}:${labelSeq}:${nanoid()}`;
|
|
1695
1722
|
const labelRuntimeScope = resolveLangfuseRuntimeScope({
|
|
1696
1723
|
runLangfuse: this.langfuse,
|
|
1697
1724
|
langfuseOverlay: labelContext?.langfuse,
|
|
1698
1725
|
traceIdSeed: labelTraceSeed,
|
|
1726
|
+
runId: labelScopeRunId,
|
|
1699
1727
|
});
|
|
1700
1728
|
/** Handler only when a session id resolved from
|
|
1701
1729
|
* `chainOptions.configurable.thread_id`: without it the label call has
|
|
@@ -1716,6 +1744,9 @@ export class Run<_T extends t.BaseGraphState> {
|
|
|
1716
1744
|
labelLangfuseConfig?.deterministicTraceId === true
|
|
1717
1745
|
? labelTraceSeed
|
|
1718
1746
|
: undefined,
|
|
1747
|
+
runId: labelScopeRunId,
|
|
1748
|
+
toolOutputTracing: labelRuntimeScope.toolOutputTracing,
|
|
1749
|
+
traceName: labelChainOptions.runName ?? labelRunName,
|
|
1719
1750
|
});
|
|
1720
1751
|
}
|
|
1721
1752
|
if (labelLangfuseHandler != null) {
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
[
|
|
2
|
+
{
|
|
3
|
+
"id": "sandbox-python-version",
|
|
4
|
+
"traceId": "eeab6f47000fe2a6aac76cde6bbc67fd",
|
|
5
|
+
"prompt": "Reasoning excerpts:\n- I'll work through each check methodically, running them one at a time and documenting what I find.\n\nTool calls:\n- run_tools_with_bash({\"code\":\"python3 --version\"}) → stdout:\nPython 3.14.4\n\nLabel:",
|
|
6
|
+
"productionLabel": "Confirmed Python 3.14.4 is installed"
|
|
7
|
+
},
|
|
8
|
+
{
|
|
9
|
+
"id": "mnt-data-write-test",
|
|
10
|
+
"traceId": "8ee41468850d42e5477af07759519e84",
|
|
11
|
+
"prompt": "Intent (assistant's last message): **Python version: 3.14.4**\n\nNow let's check if files in /mnt/data survive between calls.\n\nTool calls:\n- bash_tool({\"command\":\"echo \\\"hello from probe 1, written at $(date)\\\" > /mnt/data/persist_test.txt; cat /mnt/data/persist_test.txt\"}) → stdout:\nhello from probe 1, written at Wed Jul 29 11:30:09 AM UTC 2026\n\nLabel:",
|
|
12
|
+
"productionLabel": "Wrote test file to /mnt/data, confirmed persistence"
|
|
13
|
+
},
|
|
14
|
+
{
|
|
15
|
+
"id": "mnt-data-persistence",
|
|
16
|
+
"traceId": "64b318bd7251b66baccd75009d947682",
|
|
17
|
+
"prompt": "Intent (assistant's last message): **Python version: 3.14.4**\n\nNow let's check if files in /mnt/data survive between calls.\n\nReasoning excerpts:\n- I should make a second request to verify the behavior persists across multiple calls.\n\nTool calls:\n- bash_tool({\"command\":\"ls -la /mnt/data/persist_test.txt && cat /mnt/data/persist_test.txt\"}) → stdout:\n-rw------- 1 65534 65534 63 Jul 29 11:30 /mnt/data/persist_test.txt\nhello from probe 1, written at Wed Jul 29 11:30:09 AM UTC 2026\n\nLabel:",
|
|
18
|
+
"productionLabel": "Confirmed /mnt/data persists between tool calls"
|
|
19
|
+
},
|
|
20
|
+
{
|
|
21
|
+
"id": "mnt-data-tmpfs",
|
|
22
|
+
"traceId": "bff7f9766cc2916230403a04c47c92af",
|
|
23
|
+
"prompt": "Intent (assistant's last message): **Files in /mnt/data DO survive between calls** — the file I wrote in the previous, separate sandbox invocation is still there with its original content intact.\n\nNow let's check RAM and disk.\n\nTool calls:\n- bash_tool({\"command\":\"echo \\\"=== MEMORY ===\\\"; free -h; echo; echo \\\"=== DISK ===\\\"; df -h /mnt/data / /tmp 2>/dev/null\"}) → stdout:\n=== MEMORY ===\n\n=== DISK ===\nFilesystem Size Used Avail Use% Mounted on\ntmpfs 1.0G 32K 1.0G 1% /mnt/data\nnone 16M 0 16M 0% /\nnone 20M 0 20M 0% /tmp\n\nstderr:\nfree: Unable to create meminfo structure\nNote: /tmp files are same-call scratch only and were not persisted; use /mnt/data for files needed later.\n\nLabel:",
|
|
24
|
+
"productionLabel": "Confirmed /mnt/data is tmpfs with 1GB capacity"
|
|
25
|
+
},
|
|
26
|
+
{
|
|
27
|
+
"id": "cpu-meminfo-disk",
|
|
28
|
+
"traceId": "6faf26136e125c29fe0bae7ddd87c59e",
|
|
29
|
+
"prompt": "Intent (assistant's last message): `free` failed, let me try /proc/meminfo directly — and note disk is quite constrained (tmpfs-based, tiny root/tmp).\n\nTool calls:\n- bash_tool({\"command\":\"echo \\\"=== /proc/meminfo ===\\\"; cat /proc/meminfo 2>/dev/null | head -5\\necho\\necho \\\"=== nproc ===\\\"; nproc\\necho\\necho \\\"=== /proc/cpuinfo count ===\\\"; grep -c ^processor /proc/cpuinfo 2>/dev/null\\necho\\necho \\\"=== disk full df ===\\\"; df -h 2>/dev/null\"}) → stdout:\n=== /proc/meminfo ===\n\n=== nproc ===\n1\n\n=== /proc/cpuinfo count ===\n0\n\n=== disk full df ===\nFilesystem Size Used Avail Use% Mounted on\nnone 16M 0 16M 0% /\n/dev/vdc 16G 4.4G 11G 30% /usr\nnone 16M 4.0K 16M 1% /etc/hosts\ntmpfs 64M 0 64M 0% /dev/null\nnone 20M 0 20M 0% /tmp\ntmpfs 1.0G 32K 1.0G 1% /mnt/data\n\nLabel:",
|
|
30
|
+
"productionLabel": "Confirmed single CPU, /proc/meminfo empty, disk constrained with 16M root and 20M tmp"
|
|
31
|
+
},
|
|
32
|
+
{
|
|
33
|
+
"id": "rlimit-cgroup",
|
|
34
|
+
"traceId": "2797232991aa7e841081a6b6175bd62b",
|
|
35
|
+
"prompt": "Intent (assistant's last message): `/proc/meminfo` is empty/restricted, but I can check cgroup limits for a real memory figure.\n\nTool calls:\n- bash_tool({\"command\":\"echo \\\"=== cgroup memory limit ===\\\"\\nfor f in /sys/fs/cgroup/memory.max /sys/fs/cgroup/memory/memory.limit_in_bytes; do\\n [ -f \\\"$f\\\" ] && echo \\\"$f: $(cat $f)\\\"\\ndone\\necho\\necho \\\"=== cgroup cpu ===\\\"\\nfor f in /sys/fs/cgroup/cpu.max /sys/fs/cgroup/cpu/cpu.cfs_quota_us; do\\n [ -f \\\"$f\\\" ] && echo \\\"$f: $(cat $f)\\\"\\ndone\\necho\\npython3 -c \\\"import resource; print('soft/hard RLIMIT_AS:', resource.getrlimit(resource.RLIMIT_AS))\\\" 2>/dev/null\"}) → stdout:\n=== cgroup memory limit ===\n\n=== cgroup cpu ===\n\nsoft/hard RLIMIT_AS: (17179869184, 17179869184)\n\nLabel:",
|
|
36
|
+
"productionLabel": "Found RLIMIT_AS ceiling at 16GB, cgroup limits unavailable"
|
|
37
|
+
},
|
|
38
|
+
{
|
|
39
|
+
"id": "network-tools",
|
|
40
|
+
"traceId": "e813a6ee932da05d573ced56424265b8",
|
|
41
|
+
"prompt": "Intent (assistant's last message): No cgroup files readable, but RLIMIT_AS shows a **16 GiB address-space limit** — likely the effective memory ceiling. Now let's check network access.\n\nTool calls:\n- bash_tool({\"command\":\"echo \\\"=== ping ===\\\"; timeout 3 ping -c1 8.8.8.8 2>&1\\necho\\necho \\\"=== curl ===\\\"; timeout 5 curl -sS -o /dev/null -w \\\"HTTP_CODE:%{http_code}\\\\n\\\" https://www.google.com 2>&1\\necho\\necho \\\"=== curl example.com ===\\\"; timeout 5 curl -sS -o /dev/null -w \\\"HTTP_CODE:%{http_code}\\\\n\\\" http://example.com 2>&1\\necho\\necho \\\"=== DNS resolution ===\\\"; timeout 3 getent hosts google.com 2>&1\"}) → stdout:\n=== ping ===\ntimeout: failed to run command ‘ping’: No such file or directory\n\n=== curl ===\ntimeout: failed to run command ‘curl’: No such file or directory\n\n=== curl example.com ===\ntimeout: failed to run command ‘curl’: No such file or directory\n\n=== DNS resolution ===\n\nLabel:",
|
|
42
|
+
"productionLabel": "Network tools unavailable, DNS resolution failed"
|
|
43
|
+
},
|
|
44
|
+
{
|
|
45
|
+
"id": "dns-tcp",
|
|
46
|
+
"traceId": "d96b49502c4a0ecb4c9325c2e74b43bf",
|
|
47
|
+
"prompt": "Intent (assistant's last message): No curl/ping binaries — let me test connectivity directly via Python sockets/urllib instead.\n\nTool calls:\n- bash_tool({\"command\":\"python3 - <<'EOF'\\nimport socket, urllib.request, time\\n\\n# DNS test\\ntry:\\n ip = socket.gethostbyname(\\\"example.com\\\")\\n print(\\\"DNS resolution OK:\\\", ip)\\nexcept Exception as e:\\n print(\\\"DNS resolution FAILED:\\\", repr(e))\\n\\n# raw TCP connect test\\ntry:\\n s = socket.create_connection((\\\"8.8.8.8\\\", 53), timeout=4)\\n print(\\\"TCP connect to 8.8.8.8:53 OK\\\")\\n s.close()\\nexcept Exception as e:\\n print(\\\"TCP connect FAILED:\\\", repr(e))\\n\\n# HTTP fetch test\\ntry:\\n t0 = time.time()\\n r = urllib.request.urlopen(\\\"http://example.com\\\", timeout=5)\\n print(\\…) → stdout:\nDNS resolution FAILED: gaierror(-2, 'Name or service not known')\nTCP connect FAILED: PermissionError(1, 'Operation not permitted')\nHTTP fetch FAILED: URLError(gaierror(-3, 'Temporary failure in name resolution'))\n\nLabel:",
|
|
48
|
+
"productionLabel": "DNS and TCP connectivity both blocked or unavailable"
|
|
49
|
+
},
|
|
50
|
+
{
|
|
51
|
+
"id": "python-packages",
|
|
52
|
+
"traceId": "a6fcd0fd40b490285ced22c236023e2c",
|
|
53
|
+
"prompt": "Intent (assistant's last message): **No network access** — DNS resolution fails and raw TCP connections are blocked with a permission error. This sandbox is fully network-isolated.\n\nNow let's check for numpy/pandas/scipy/torch.\n\nTool calls:\n- bash_tool({\"command\":\"python3 - <<'EOF'\\nmods = [\\\"numpy\\\", \\\"pandas\\\", \\\"scipy\\\", \\\"torch\\\"]\\nfor m in mods:\\n try:\\n mod = __import__(m)\\n ver = getattr(mod, \\\"__version__\\\", \\\"unknown\\\")\\n print(f\\\"{m}: INSTALLED, version {ver}\\\")\\n except ImportError as e:\\n print(f\\\"{m}: NOT INSTALLED ({e})\\\")\\nEOF\"}) → stdout:\nnumpy: INSTALLED, version 2.4.6\npandas: INSTALLED, version 2.3.3\nscipy: INSTALLED, version 1.18.0\ntorch: NOT INSTALLED (No module named 'torch')\n\nLabel:",
|
|
54
|
+
"productionLabel": "Confirmed numpy, pandas, scipy installed; torch unavailable"
|
|
55
|
+
}
|
|
56
|
+
]
|
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Mechanical label checks. These catch format violations and the two
|
|
3
|
+
* measurable prose failures (register collapse via first-word tallies,
|
|
4
|
+
* cross-batch redundancy via content-word overlap); commit-log READABILITY
|
|
5
|
+
* still needs the human pass over results/latest.md.
|
|
6
|
+
*
|
|
7
|
+
* Ported from LibreChat #14527 with two divergences, both backport
|
|
8
|
+
* candidates for LibreChat's scripts/activity-labels/checks.js: tool-echo
|
|
9
|
+
* matching normalizes namespaced MCP names (see the candidate-set comment
|
|
10
|
+
* below), and COUNT_ECHO also catches spelled-out counts.
|
|
11
|
+
*/
|
|
12
|
+
const STOPWORDS = new Set([
|
|
13
|
+
'the',
|
|
14
|
+
'a',
|
|
15
|
+
'an',
|
|
16
|
+
'to',
|
|
17
|
+
'of',
|
|
18
|
+
'and',
|
|
19
|
+
'or',
|
|
20
|
+
'with',
|
|
21
|
+
'for',
|
|
22
|
+
'in',
|
|
23
|
+
'on',
|
|
24
|
+
'at',
|
|
25
|
+
'is',
|
|
26
|
+
'are',
|
|
27
|
+
'was',
|
|
28
|
+
'were',
|
|
29
|
+
'be',
|
|
30
|
+
'been',
|
|
31
|
+
'it',
|
|
32
|
+
'its',
|
|
33
|
+
'as',
|
|
34
|
+
'by',
|
|
35
|
+
'from',
|
|
36
|
+
'that',
|
|
37
|
+
'this',
|
|
38
|
+
'both',
|
|
39
|
+
'all',
|
|
40
|
+
'no',
|
|
41
|
+
'not',
|
|
42
|
+
'via',
|
|
43
|
+
]);
|
|
44
|
+
|
|
45
|
+
function words(label) {
|
|
46
|
+
return label.trim().split(/\s+/).filter(Boolean);
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
const SUFFIXES = [
|
|
50
|
+
'ations',
|
|
51
|
+
'ation',
|
|
52
|
+
'ence',
|
|
53
|
+
'ance',
|
|
54
|
+
'ings',
|
|
55
|
+
'ing',
|
|
56
|
+
'ed',
|
|
57
|
+
'es',
|
|
58
|
+
's',
|
|
59
|
+
];
|
|
60
|
+
|
|
61
|
+
/** Crude suffix stemmer so persists/persistence/persisted collide — enough
|
|
62
|
+
* for overlap detection; linguistic correctness is not the goal. */
|
|
63
|
+
function stem(word) {
|
|
64
|
+
if (word.length < 5) {
|
|
65
|
+
return word;
|
|
66
|
+
}
|
|
67
|
+
for (const suffix of SUFFIXES) {
|
|
68
|
+
if (word.endsWith(suffix) && word.length - suffix.length >= 4) {
|
|
69
|
+
return word.slice(0, word.length - suffix.length);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
return word;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function contentWords(label) {
|
|
76
|
+
return words(label.toLowerCase().replace(/[^a-z0-9/._-]+/g, ' '))
|
|
77
|
+
.filter((word) => !STOPWORDS.has(word))
|
|
78
|
+
.map(stem);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/** Payload tokens carry the informative delta between template-shaped
|
|
82
|
+
* labels: numbers, versions, paths, filenames. */
|
|
83
|
+
function isPayload(word) {
|
|
84
|
+
return /\d/.test(word) || word.includes('/') || word.includes('.');
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
function jaccard(a, b) {
|
|
88
|
+
const setA = new Set(a);
|
|
89
|
+
const setB = new Set(b);
|
|
90
|
+
if (setA.size === 0 || setB.size === 0) {
|
|
91
|
+
return 0;
|
|
92
|
+
}
|
|
93
|
+
let intersection = 0;
|
|
94
|
+
for (const word of setA) {
|
|
95
|
+
if (setB.has(word)) {
|
|
96
|
+
intersection += 1;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
return intersection / (setA.size + setB.size - intersection);
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
const GENERIC_OPENER = /^(ran|used|executed|called|invoked|performed)\b/i;
|
|
103
|
+
/** Digits or the spelled-out counts a label-length line can fit — the
|
|
104
|
+
* instruction forbids counting calls either way. */
|
|
105
|
+
const COUNT_ECHO =
|
|
106
|
+
/\b(\d+|one|two|three|four|five|six|seven|eight|nine|ten|eleven|twelve)\s+(tools?|commands?|calls?)\b/i;
|
|
107
|
+
const DUP_THRESHOLD = 0.5;
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* @param label generated label text
|
|
111
|
+
* @param entries the batch's tool entries (for tool-name echo detection)
|
|
112
|
+
* @param previousLabels labels generated EARLIER in the same case chain,
|
|
113
|
+
* regardless of whether the variant saw them — redundancy is measured
|
|
114
|
+
* uniformly so continuity variants can be compared against blind ones.
|
|
115
|
+
*/
|
|
116
|
+
function checkLabel(label, { entries = [], previousLabels = [] } = {}) {
|
|
117
|
+
const flags = [];
|
|
118
|
+
const wordList = words(label);
|
|
119
|
+
if (wordList.length < 4 || wordList.length > 9) {
|
|
120
|
+
flags.push(`len:${wordList.length}`);
|
|
121
|
+
}
|
|
122
|
+
if (/[.!?,;:]$/.test(label.trim())) {
|
|
123
|
+
flags.push('punct');
|
|
124
|
+
}
|
|
125
|
+
if (/^["'`]|["'`]$/.test(label.trim())) {
|
|
126
|
+
flags.push('quote');
|
|
127
|
+
}
|
|
128
|
+
if (/[*`]|^#|\[.*\]\(/.test(label)) {
|
|
129
|
+
flags.push('md');
|
|
130
|
+
}
|
|
131
|
+
if (GENERIC_OPENER.test(label.trim())) {
|
|
132
|
+
flags.push('opener');
|
|
133
|
+
}
|
|
134
|
+
const lower = label.toLowerCase();
|
|
135
|
+
for (const entry of entries) {
|
|
136
|
+
const name = String(entry.toolName ?? '').toLowerCase();
|
|
137
|
+
if (name.length <= 3) {
|
|
138
|
+
continue;
|
|
139
|
+
}
|
|
140
|
+
/** Namespaced MCP names (`mcp__github__search_repositories`) would
|
|
141
|
+
* otherwise normalize their double underscores to double spaces and
|
|
142
|
+
* never match a natural echo; collapse separator runs and also try
|
|
143
|
+
* the post-namespace tail ("search repositories"). Only multi-char
|
|
144
|
+
* candidates derived from the full name — never single middle
|
|
145
|
+
* segments like "github", which are legitimate label subjects. */
|
|
146
|
+
const segments = name.split('__');
|
|
147
|
+
const tail = segments[segments.length - 1];
|
|
148
|
+
const candidates = new Set([
|
|
149
|
+
name,
|
|
150
|
+
name.replace(/_+/g, ' '),
|
|
151
|
+
tail,
|
|
152
|
+
tail.replace(/_/g, ' '),
|
|
153
|
+
]);
|
|
154
|
+
let echoed = false;
|
|
155
|
+
for (const candidate of candidates) {
|
|
156
|
+
if (candidate.length > 3 && lower.includes(candidate)) {
|
|
157
|
+
echoed = true;
|
|
158
|
+
break;
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
if (echoed) {
|
|
162
|
+
flags.push(`tool-echo:${entry.toolName}`);
|
|
163
|
+
break;
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
if (COUNT_ECHO.test(label)) {
|
|
167
|
+
flags.push('count-echo');
|
|
168
|
+
}
|
|
169
|
+
/** Overlap splits into two flags: `restate` (high overlap, no payload
|
|
170
|
+
* delta — the line adds nothing over a previous header; the production
|
|
171
|
+
* 2/3 and 7/8 failure) and `template` (high overlap but the differing
|
|
172
|
+
* tokens are numbers/paths — same sentence frame, new information, e.g.
|
|
173
|
+
* fib(1)→fib(2). Often fine, arguably better than synonym churn). */
|
|
174
|
+
const own = contentWords(label);
|
|
175
|
+
let maxOverlap = 0;
|
|
176
|
+
let worst = null;
|
|
177
|
+
for (const previous of previousLabels) {
|
|
178
|
+
const other = contentWords(previous);
|
|
179
|
+
const overlap = jaccard(own, other);
|
|
180
|
+
if (overlap > maxOverlap) {
|
|
181
|
+
maxOverlap = overlap;
|
|
182
|
+
worst = other;
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
if (maxOverlap > DUP_THRESHOLD && worst != null) {
|
|
186
|
+
const otherSet = new Set(worst);
|
|
187
|
+
const ownSet = new Set(own);
|
|
188
|
+
const differing = [
|
|
189
|
+
...own.filter((word) => !otherSet.has(word)),
|
|
190
|
+
...worst.filter((word) => !ownSet.has(word)),
|
|
191
|
+
];
|
|
192
|
+
const informativeDelta = differing.some(isPayload);
|
|
193
|
+
flags.push(
|
|
194
|
+
`${informativeDelta ? 'template' : 'restate'}:${maxOverlap.toFixed(2)}`
|
|
195
|
+
);
|
|
196
|
+
}
|
|
197
|
+
return {
|
|
198
|
+
flags,
|
|
199
|
+
wordCount: wordList.length,
|
|
200
|
+
firstWord: wordList[0] ?? '',
|
|
201
|
+
maxOverlap,
|
|
202
|
+
};
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
module.exports = { checkLabel, contentWords, jaccard, words, stem };
|