@librechat/agents 3.2.64 → 3.2.66
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/graphs/Graph.cjs +18 -2
- package/dist/cjs/graphs/Graph.cjs.map +1 -1
- package/dist/cjs/instrumentation.cjs +15 -3
- package/dist/cjs/instrumentation.cjs.map +1 -1
- package/dist/cjs/langfuseToolOutputTracing.cjs +1 -2
- package/dist/cjs/langfuseToolOutputTracing.cjs.map +1 -1
- package/dist/cjs/langfuseTraceShaping.cjs +51 -24
- package/dist/cjs/langfuseTraceShaping.cjs.map +1 -1
- package/dist/cjs/llm/anthropic/index.cjs +73 -9
- package/dist/cjs/llm/anthropic/index.cjs.map +1 -1
- package/dist/cjs/llm/anthropic/types.cjs.map +1 -1
- package/dist/cjs/llm/anthropic/utils/message_inputs.cjs +49 -9
- package/dist/cjs/llm/anthropic/utils/message_inputs.cjs.map +1 -1
- package/dist/cjs/llm/anthropic/utils/message_outputs.cjs +3 -1
- package/dist/cjs/llm/anthropic/utils/message_outputs.cjs.map +1 -1
- package/dist/cjs/llm/anthropic/utils/stream_events.cjs +337 -0
- package/dist/cjs/llm/anthropic/utils/stream_events.cjs.map +1 -0
- package/dist/cjs/llm/bedrock/utils/message_inputs.cjs +8 -0
- package/dist/cjs/llm/bedrock/utils/message_inputs.cjs.map +1 -1
- package/dist/cjs/tools/BashExecutor.cjs +9 -8
- package/dist/cjs/tools/BashExecutor.cjs.map +1 -1
- package/dist/cjs/tools/CodeExecutor.cjs +9 -7
- package/dist/cjs/tools/CodeExecutor.cjs.map +1 -1
- package/dist/cjs/tools/search/crw-scraper.cjs +165 -0
- package/dist/cjs/tools/search/crw-scraper.cjs.map +1 -0
- package/dist/cjs/tools/search/crw-search.cjs +105 -0
- package/dist/cjs/tools/search/crw-search.cjs.map +1 -0
- package/dist/cjs/tools/search/search.cjs +4 -2
- package/dist/cjs/tools/search/search.cjs.map +1 -1
- package/dist/cjs/tools/search/tool.cjs +15 -3
- package/dist/cjs/tools/search/tool.cjs.map +1 -1
- package/dist/esm/graphs/Graph.mjs +18 -2
- package/dist/esm/graphs/Graph.mjs.map +1 -1
- package/dist/esm/instrumentation.mjs +15 -3
- package/dist/esm/instrumentation.mjs.map +1 -1
- package/dist/esm/langfuseToolOutputTracing.mjs +1 -2
- package/dist/esm/langfuseToolOutputTracing.mjs.map +1 -1
- package/dist/esm/langfuseTraceShaping.mjs +51 -24
- package/dist/esm/langfuseTraceShaping.mjs.map +1 -1
- package/dist/esm/llm/anthropic/index.mjs +73 -9
- package/dist/esm/llm/anthropic/index.mjs.map +1 -1
- package/dist/esm/llm/anthropic/types.mjs.map +1 -1
- package/dist/esm/llm/anthropic/utils/message_inputs.mjs +49 -9
- package/dist/esm/llm/anthropic/utils/message_inputs.mjs.map +1 -1
- package/dist/esm/llm/anthropic/utils/message_outputs.mjs +3 -2
- package/dist/esm/llm/anthropic/utils/message_outputs.mjs.map +1 -1
- package/dist/esm/llm/anthropic/utils/stream_events.mjs +337 -0
- package/dist/esm/llm/anthropic/utils/stream_events.mjs.map +1 -0
- package/dist/esm/llm/bedrock/utils/message_inputs.mjs +8 -0
- package/dist/esm/llm/bedrock/utils/message_inputs.mjs.map +1 -1
- package/dist/esm/tools/BashExecutor.mjs +9 -8
- package/dist/esm/tools/BashExecutor.mjs.map +1 -1
- package/dist/esm/tools/CodeExecutor.mjs +9 -7
- package/dist/esm/tools/CodeExecutor.mjs.map +1 -1
- package/dist/esm/tools/search/crw-scraper.mjs +163 -0
- package/dist/esm/tools/search/crw-scraper.mjs.map +1 -0
- package/dist/esm/tools/search/crw-search.mjs +103 -0
- package/dist/esm/tools/search/crw-search.mjs.map +1 -0
- package/dist/esm/tools/search/search.mjs +4 -2
- package/dist/esm/tools/search/search.mjs.map +1 -1
- package/dist/esm/tools/search/tool.mjs +15 -3
- package/dist/esm/tools/search/tool.mjs.map +1 -1
- package/dist/types/graphs/Graph.d.ts +2 -0
- package/dist/types/langfuseTraceShaping.d.ts +12 -6
- package/dist/types/llm/anthropic/index.d.ts +2 -0
- package/dist/types/llm/anthropic/types.d.ts +2 -0
- package/dist/types/llm/anthropic/utils/message_outputs.d.ts +1 -2
- package/dist/types/llm/anthropic/utils/stream_events.d.ts +25 -0
- package/dist/types/tools/BashExecutor.d.ts +7 -6
- package/dist/types/tools/CodeExecutor.d.ts +7 -5
- package/dist/types/tools/search/crw-scraper.d.ts +41 -0
- package/dist/types/tools/search/crw-search.d.ts +4 -0
- package/dist/types/tools/search/types.d.ts +88 -3
- package/dist/types/types/graph.d.ts +10 -3
- package/package.json +1 -1
- package/src/graphs/Graph.ts +36 -3
- package/src/instrumentation.ts +20 -0
- package/src/langfuseToolOutputTracing.ts +2 -4
- package/src/langfuseTraceShaping.ts +73 -20
- package/src/llm/anthropic/index.ts +134 -10
- package/src/llm/anthropic/inherited-content-utils.spec.ts +10 -5
- package/src/llm/anthropic/inherited-stream-events.spec.ts +513 -9
- package/src/llm/anthropic/llm.spec.ts +100 -16
- package/src/llm/anthropic/types.ts +3 -0
- package/src/llm/anthropic/utils/cross-provider-server-tools.test.ts +110 -0
- package/src/llm/anthropic/utils/message_inputs.ts +75 -3
- package/src/llm/anthropic/utils/message_outputs.ts +10 -2
- package/src/llm/anthropic/utils/stream_events.ts +471 -0
- package/src/llm/bedrock/utils/cross-provider-server-tools.test.ts +122 -0
- package/src/llm/bedrock/utils/message_inputs.ts +13 -0
- package/src/specs/langfuse-instrumentation.test.ts +64 -0
- package/src/specs/langfuse-routing.integration.test.ts +49 -4
- package/src/specs/langfuse-tool-output-tracing.test.ts +5 -4
- package/src/specs/langfuse-trace-shaping.test.ts +80 -9
- package/src/specs/subagent.test.ts +180 -0
- package/src/tools/BashExecutor.ts +9 -8
- package/src/tools/CodeExecutor.ts +9 -7
- package/src/tools/__tests__/BashExecutor.test.ts +16 -5
- package/src/tools/__tests__/CodeExecutor.stateful.test.ts +17 -6
- package/src/tools/search/crw-scraper.ts +244 -0
- package/src/tools/search/crw-search.ts +167 -0
- package/src/tools/search/crw.test.ts +836 -0
- package/src/tools/search/search.ts +7 -1
- package/src/tools/search/tool.ts +23 -3
- package/src/tools/search/types.ts +103 -3
- package/src/types/graph.ts +10 -3
|
@@ -45,15 +45,16 @@ Usage:
|
|
|
45
45
|
- NEVER use this tool to execute malicious commands.
|
|
46
46
|
`.trim();
|
|
47
47
|
/**
|
|
48
|
-
* Bash statefulness is filesystem-tier
|
|
49
|
-
*
|
|
50
|
-
*
|
|
51
|
-
*
|
|
52
|
-
*
|
|
48
|
+
* Bash statefulness is filesystem-tier and scoped to `/mnt/data`. The machine
|
|
49
|
+
* is warm across calls, but each call runs in a fresh sandbox (new process
|
|
50
|
+
* tree + private /tmp), so background processes are reaped when the call ends
|
|
51
|
+
* and anything written outside /mnt/data is discarded. The note must not
|
|
52
|
+
* promise otherwise: a model told background processes survive will start a
|
|
53
|
+
* server in one call and assume it is listening in the next.
|
|
53
54
|
*/
|
|
54
|
-
const STATEFUL_BASH_NOTE = "Session state
|
|
55
|
+
const STATEFUL_BASH_NOTE = "Session state: commands in this conversation run on the same warm machine, so files written to /mnt/data persist between calls. Each call runs in a fresh, isolated sandbox: shell variables, the working directory, /tmp, and background processes do NOT survive after the call returns — a process started in one call is terminated when that call ends. Only /mnt/data is durable (the machine itself may also be reset at any time).";
|
|
55
56
|
const StatefulBashExecutionToolDescription = `
|
|
56
|
-
Runs bash commands and returns stdout/stderr output
|
|
57
|
+
Runs bash commands and returns stdout/stderr output. Commands in this conversation share one warm machine with a persistent /mnt/data, but each command runs in its own isolated sandbox (not a persistent shell session).
|
|
57
58
|
|
|
58
59
|
${STATEFUL_BASH_NOTE}
|
|
59
60
|
|
|
@@ -97,7 +98,7 @@ function buildBashExecutionToolDescription(options) {
|
|
|
97
98
|
return base;
|
|
98
99
|
}
|
|
99
100
|
const STATELESS_BASH_PARAM_NOTE = "The environment is stateless; variables and state don't persist between executions.";
|
|
100
|
-
const STATEFUL_BASH_PARAM_NOTE = "Files
|
|
101
|
+
const STATEFUL_BASH_PARAM_NOTE = "Files written to /mnt/data persist between calls on the same warm machine. Each call runs in a fresh sandbox: shell variables, cwd, /tmp, and background processes do NOT survive the call. Only /mnt/data is durable.";
|
|
101
102
|
function buildBashExecutionToolSchema(opts) {
|
|
102
103
|
const note = opts?.statefulSessions === true ? STATEFUL_BASH_PARAM_NOTE : STATELESS_BASH_PARAM_NOTE;
|
|
103
104
|
const commandDescription = BashExecutionToolSchema.properties.command.description.replace(STATELESS_BASH_PARAM_NOTE, note);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"BashExecutor.cjs","names":["getCodeBaseURL","CODE_ARTIFACT_PATH_GUIDANCE","BASH_SHELL_GUIDANCE","resolveCodeApiAuthHeaders","HttpsProxyAgent","buildCodeApiHttpErrorMessage","emptyOutputMessage","appendTmpScratchReminder","appendCodeSessionFileSummary","appendFailedExecutionFileReminder"],"sources":["../../../src/tools/BashExecutor.ts"],"sourcesContent":["import { config } from 'dotenv';\nimport fetch, { RequestInit } from 'node-fetch';\nimport { HttpsProxyAgent } from 'https-proxy-agent';\nimport { tool, DynamicStructuredTool } from '@langchain/core/tools';\nimport type * as t from '@/types';\nimport {\n BASH_SHELL_GUIDANCE,\n CODE_ARTIFACT_PATH_GUIDANCE,\n appendFailedExecutionFileReminder,\n appendTmpScratchReminder,\n appendCodeSessionFileSummary,\n emptyOutputMessage,\n buildCodeApiHttpErrorMessage,\n getCodeBaseURL,\n resolveCodeApiAuthHeaders,\n} from './CodeExecutor';\nimport { Constants } from '@/common';\n\nconfig();\n\nconst baseEndpoint = getCodeBaseURL();\nconst EXEC_ENDPOINT = `${baseEndpoint}/exec`;\n\nexport const BashExecutionToolSchema = {\n type: 'object',\n properties: {\n command: {\n type: 'string',\n description: `The bash command or script to execute.\n- The environment is stateless; variables and state don't persist between executions.\n- Prior /mnt/data files are available and can be modified in place.\n- ${CODE_ARTIFACT_PATH_GUIDANCE}\n- ${BASH_SHELL_GUIDANCE}\n- Input code **IS ALREADY** displayed to the user, so **DO NOT** repeat it in your response unless asked.\n- Output code **IS NOT** displayed to the user, so **DO** write all desired output explicitly.\n- IMPORTANT: You MUST explicitly print/output ALL results you want the user to see.\n- Use \\`echo\\`, \\`printf\\`, or \\`cat\\` for all outputs.`,\n },\n args: {\n type: 'array',\n items: { type: 'string' },\n description:\n 'Additional arguments to execute the command with. This should only be used if the input command requires additional arguments to run.',\n },\n },\n required: ['command'],\n} as const;\n\nexport const BashExecutionToolDescription = `\nRuns bash commands and returns stdout/stderr output from a stateless execution environment, similar to running scripts in a command-line interface. Each execution is isolated and independent.\n\nUsage:\n- No network access available.\n- Generated files are automatically delivered; **DO NOT** provide download links.\n- ${CODE_ARTIFACT_PATH_GUIDANCE}\n- ${BASH_SHELL_GUIDANCE}\n- NEVER use this tool to execute malicious commands.\n`.trim();\n\n/**\n * Bash statefulness is filesystem-tier: on a warm session the machine (files\n * including /tmp, installed packages, background processes) persists between\n * calls, but each call may start a fresh shell — so shell variables and cwd\n * are NOT reliable, and the machine can be reset at any time. Only /mnt/data\n * is durable.\n */\nexport const STATEFUL_BASH_NOTE =\n 'Session state (best-effort): commands in this conversation usually run on the same machine, so files (including /tmp), installed packages, and running background processes from earlier calls typically persist. Each call may still start a fresh shell — do not rely on shell variables or the working directory carrying over — and the machine may be reset at any time. Only /mnt/data is durable.';\n\nexport const StatefulBashExecutionToolDescription = `\nRuns bash commands and returns stdout/stderr output from a session-based execution environment, similar to a long-running machine.\n\n${STATEFUL_BASH_NOTE}\n\nUsage:\n- No network access available.\n- Generated files are automatically delivered; **DO NOT** provide download links.\n- ${CODE_ARTIFACT_PATH_GUIDANCE}\n- ${BASH_SHELL_GUIDANCE}\n- NEVER use this tool to execute malicious commands.\n`.trim();\n\n/**\n * Supplemental prompt documenting the tool-output reference feature.\n *\n * Hosts should append this (separated by a blank line) to the base\n * {@link BashExecutionToolDescription} only when\n * `RunConfig.toolOutputReferences.enabled` is `true`. When the feature\n * is disabled, including this text would tell the LLM to emit\n * `{{tool0turn0}}` placeholders that pass through unsubstituted and\n * leak into the shell.\n */\nexport const BashToolOutputReferencesGuide = `\nReferencing previous tool outputs:\n- Every successful tool result is tagged with a reference key of the form \\`tool<idx>turn<turn>\\` (e.g., \\`tool0turn0\\`). The key appears either as a \\`[ref: tool0turn0]\\` prefix line or, when the output is a JSON object, as a \\`_ref\\` field on the object.\n- To pipe a previous tool output into this tool, embed the placeholder \\`{{tool<idx>turn<turn>}}\\` literally anywhere in the \\`command\\` string (or any string arg). It will be substituted with the stored output verbatim before the command runs.\n- The substituted value is the original output string (no \\`[ref: …]\\` prefix, no \\`_ref\\` key), so it is safe to pipe directly into \\`jq\\`, \\`grep\\`, \\`awk\\`, etc.\n- Example (simple ASCII output): \\`echo '{{tool0turn0}}' | jq '.foo'\\` takes the full output of the first tool from the first turn and pipes it into jq.\n- For payloads that may contain quotes, parentheses, backticks, or arbitrary bytes (random/binary data, JSON with embedded quotes, multi-line strings), prefer a quoted-delimiter heredoc over \\`echo '…'\\`. The heredoc body is not interpreted by the shell, so substituted payloads pass through unchanged.\n- Heredoc example: \\`wc -c << 'EOF'\\\\n{{tool0turn0}}\\\\nEOF\\` (the quotes around \\`'EOF'\\` disable interpolation inside the body).\n- Unknown reference keys are left in place and surfaced as \\`[unresolved refs: …]\\` after the output.\n`.trim();\n\n/**\n * Composes the bash tool description, optionally appending the\n * tool-output references guide. Hosts that enable\n * `RunConfig.toolOutputReferences` should pass `enableToolOutputReferences: true`\n * when registering the tool so the LLM learns the `{{…}}` syntax it\n * will actually be able to use.\n */\nexport function buildBashExecutionToolDescription(options?: {\n enableToolOutputReferences?: boolean;\n statefulSessions?: boolean;\n}): string {\n const base =\n options?.statefulSessions === true\n ? StatefulBashExecutionToolDescription\n : BashExecutionToolDescription;\n if (options?.enableToolOutputReferences === true) {\n return `${base}\\n\\n${BashToolOutputReferencesGuide}`;\n }\n return base;\n}\n\nconst STATELESS_BASH_PARAM_NOTE =\n 'The environment is stateless; variables and state don\\'t persist between executions.';\nconst STATEFUL_BASH_PARAM_NOTE =\n 'Files, installed packages, and background processes usually persist between calls, but each call may start a fresh shell (do not rely on shell variables or cwd) and the machine may reset. Only /mnt/data is durable.';\n\nexport function buildBashExecutionToolSchema(opts?: {\n statefulSessions?: boolean;\n}): typeof BashExecutionToolSchema {\n const note =\n opts?.statefulSessions === true\n ? STATEFUL_BASH_PARAM_NOTE\n : STATELESS_BASH_PARAM_NOTE;\n const commandDescription =\n BashExecutionToolSchema.properties.command.description.replace(\n STATELESS_BASH_PARAM_NOTE,\n note\n );\n return {\n ...BashExecutionToolSchema,\n properties: {\n ...BashExecutionToolSchema.properties,\n command: {\n ...BashExecutionToolSchema.properties.command,\n description: commandDescription,\n },\n },\n } as typeof BashExecutionToolSchema;\n}\n\nexport const BashExecutionToolName = Constants.BASH_TOOL;\n\n/**\n * Default bash tool definition using the base description.\n *\n * When `RunConfig.toolOutputReferences.enabled` is `true`, build a\n * reference-aware description with\n * {@link buildBashExecutionToolDescription}\n * (`{ enableToolOutputReferences: true }`) and construct a custom\n * definition using it — using this constant as-is leaves the LLM\n * unaware of the `{{tool<i>turn<n>}}` syntax.\n */\nexport const BashExecutionToolDefinition = {\n name: BashExecutionToolName,\n description: BashExecutionToolDescription,\n schema: BashExecutionToolSchema,\n} as const;\n\nfunction createBashExecutionTool(\n params: t.BashExecutionToolParams | null = {}\n): DynamicStructuredTool {\n return tool(\n async (rawInput, config) => {\n /* `statefulSessions` is prompt-only — keep it out of the wire body. */\n const {\n authHeaders,\n statefulSessions: _statefulSessions,\n ...executionParams\n } = params ?? {};\n void _statefulSessions;\n /* Drop any model-supplied `runtime_session_hint` from the raw args: the\n * hint must only come from ToolNode's injected `_runtime_session_hint`\n * (below), never from the tool call itself. */\n const {\n command,\n runtime_session_hint: _ignoredModelHint,\n ...rest\n } = rawInput as {\n command: string;\n runtime_session_hint?: unknown;\n args?: string[];\n };\n void _ignoredModelHint;\n const { session_id, _injected_files, _runtime_session_hint } =\n (config.toolCall ?? {}) as {\n session_id?: string;\n _injected_files?: t.CodeEnvFile[];\n _runtime_session_hint?: string;\n };\n\n const postData: Record<string, unknown> = {\n lang: 'bash',\n code: command,\n ...rest,\n ...executionParams,\n };\n\n if (\n typeof _runtime_session_hint === 'string' &&\n _runtime_session_hint !== ''\n ) {\n postData.runtime_session_hint = _runtime_session_hint;\n }\n\n /* See `CodeExecutor.ts` for the rationale — `/files/<session_id>`\n * HTTP fallback was removed because codeapi's sessionAuth requires\n * kind/id query params unavailable at this point. */\n if (_injected_files && _injected_files.length > 0) {\n postData.files = _injected_files;\n } else if (\n session_id != null &&\n session_id.length > 0 &&\n !Array.isArray(postData.files)\n ) {\n // eslint-disable-next-line no-console\n console.debug(\n `[BashExecutor] No injected files for session_id=${session_id} — exec will run without input files`\n );\n }\n\n try {\n const resolvedAuthHeaders =\n await resolveCodeApiAuthHeaders(authHeaders);\n const fetchOptions: RequestInit = {\n method: 'POST',\n headers: {\n 'Content-Type': 'application/json',\n 'User-Agent': 'LibreChat/1.0',\n ...resolvedAuthHeaders,\n },\n body: JSON.stringify(postData),\n };\n\n if (process.env.PROXY != null && process.env.PROXY !== '') {\n fetchOptions.agent = new HttpsProxyAgent(process.env.PROXY);\n }\n const response = await fetch(EXEC_ENDPOINT, fetchOptions);\n if (!response.ok) {\n throw new Error(\n await buildCodeApiHttpErrorMessage('POST', EXEC_ENDPOINT, response)\n );\n }\n\n const result: t.ExecuteResult = await response.json();\n let formattedOutput = '';\n if (result.stdout) {\n formattedOutput += `stdout:\\n${result.stdout}\\n`;\n } else {\n formattedOutput += emptyOutputMessage;\n }\n if (result.stderr) formattedOutput += `stderr:\\n${result.stderr}\\n`;\n\n const outputWithReminder = appendTmpScratchReminder(\n formattedOutput,\n command\n );\n const hasFiles = result.files != null && result.files.length > 0;\n const runtimeEcho =\n result.runtime_session_id != null\n ? {\n runtime_session_id: result.runtime_session_id,\n runtime_status: result.runtime_status,\n }\n : {};\n return [\n appendCodeSessionFileSummary(outputWithReminder, result.files),\n (hasFiles\n ? {\n session_id: result.session_id,\n files: result.files,\n ...runtimeEcho,\n }\n : {\n session_id: result.session_id,\n ...runtimeEcho,\n }) satisfies t.CodeExecutionArtifact,\n ];\n } catch (error) {\n const messageWithReminder = appendFailedExecutionFileReminder(\n (error as Error | undefined)?.message ?? '',\n command\n );\n throw new Error(`Execution error:\\n\\n${messageWithReminder}`);\n }\n },\n {\n name: BashExecutionToolName,\n description: buildBashExecutionToolDescription({\n statefulSessions: params?.statefulSessions,\n }),\n schema: buildBashExecutionToolSchema(params ?? undefined),\n responseFormat: Constants.CONTENT_AND_ARTIFACT,\n }\n );\n}\n\nexport { createBashExecutionTool };\n"],"mappings":";;;;;;;;;;;mBAkBO;AAGP,MAAM,gBAAgB,GADDA,qBAAAA,eACe,EAAE;AAEtC,MAAa,0BAA0B;CACrC,MAAM;CACN,YAAY;EACV,SAAS;GACP,MAAM;GACN,aAAa;;;IAGfC,qBAAAA,4BAA4B;IAC5BC,qBAAAA,oBAAoB;;;;;EAKpB;EACA,MAAM;GACJ,MAAM;GACN,OAAO,EAAE,MAAM,SAAS;GACxB,aACE;EACJ;CACF;CACA,UAAU,CAAC,SAAS;AACtB;AAEA,MAAa,+BAA+B;;;;;;IAMxCD,qBAAAA,4BAA4B;IAC5BC,qBAAAA,oBAAoB;;EAEtB,KAAK;;;;;;;;AASP,MAAa,qBACX;AAEF,MAAa,uCAAuC;;;EAGlD,mBAAmB;;;;;IAKjBD,qBAAAA,4BAA4B;IAC5BC,qBAAAA,oBAAoB;;EAEtB,KAAK;;;;;;;;;;;AAYP,MAAa,gCAAgC;;;;;;;;;EAS3C,KAAK;;;;;;;;AASP,SAAgB,kCAAkC,SAGvC;CACT,MAAM,OACJ,SAAS,qBAAqB,OAC1B,uCACA;CACN,IAAI,SAAS,+BAA+B,MAC1C,OAAO,GAAG,KAAK,MAAM;CAEvB,OAAO;AACT;AAEA,MAAM,4BACJ;AACF,MAAM,2BACJ;AAEF,SAAgB,6BAA6B,MAEV;CACjC,MAAM,OACJ,MAAM,qBAAqB,OACvB,2BACA;CACN,MAAM,qBACJ,wBAAwB,WAAW,QAAQ,YAAY,QACrD,2BACA,IACF;CACF,OAAO;EACL,GAAG;EACH,YAAY;GACV,GAAG,wBAAwB;GAC3B,SAAS;IACP,GAAG,wBAAwB,WAAW;IACtC,aAAa;GACf;EACF;CACF;AACF;AAEA,MAAa,wBAAA;;;;;;;;;;;AAYb,MAAa,8BAA8B;CACzC,MAAM;CACN,aAAa;CACb,QAAQ;AACV;AAEA,SAAS,wBACP,SAA2C,CAAC,GACrB;CACvB,QAAA,GAAA,sBAAA,KAAA,CACE,OAAO,UAAU,WAAW;EAE1B,MAAM,EACJ,aACA,kBAAkB,mBAClB,GAAG,oBACD,UAAU,CAAC;EAKf,MAAM,EACJ,SACA,sBAAsB,mBACtB,GAAG,SACD;EAMJ,MAAM,EAAE,YAAY,iBAAiB,0BAClC,OAAO,YAAY,CAAC;EAMvB,MAAM,WAAoC;GACxC,MAAM;GACN,MAAM;GACN,GAAG;GACH,GAAG;EACL;EAEA,IACE,OAAO,0BAA0B,YACjC,0BAA0B,IAE1B,SAAS,uBAAuB;EAMlC,IAAI,mBAAmB,gBAAgB,SAAS,GAC9C,SAAS,QAAQ;OACZ,IACL,cAAc,QACd,WAAW,SAAS,KACpB,CAAC,MAAM,QAAQ,SAAS,KAAK,GAG7B,QAAQ,MACN,mDAAmD,WAAW,qCAChE;EAGF,IAAI;GAGF,MAAM,eAA4B;IAChC,QAAQ;IACR,SAAS;KACP,gBAAgB;KAChB,cAAc;KACd,GAAG,MANCC,qBAAAA,0BAA0B,WAAW;IAO3C;IACA,MAAM,KAAK,UAAU,QAAQ;GAC/B;GAEA,IAAI,QAAQ,IAAI,SAAS,QAAQ,QAAQ,IAAI,UAAU,IACrD,aAAa,QAAQ,IAAIC,kBAAAA,gBAAgB,QAAQ,IAAI,KAAK;GAE5D,MAAM,WAAW,OAAA,GAAA,WAAA,QAAA,CAAY,eAAe,YAAY;GACxD,IAAI,CAAC,SAAS,IACZ,MAAM,IAAI,MACR,MAAMC,qBAAAA,6BAA6B,QAAQ,eAAe,QAAQ,CACpE;GAGF,MAAM,SAA0B,MAAM,SAAS,KAAK;GACpD,IAAI,kBAAkB;GACtB,IAAI,OAAO,QACT,mBAAmB,YAAY,OAAO,OAAO;QAE7C,mBAAmBC,qBAAAA;GAErB,IAAI,OAAO,QAAQ,mBAAmB,YAAY,OAAO,OAAO;GAEhE,MAAM,qBAAqBC,qBAAAA,yBACzB,iBACA,OACF;GACA,MAAM,WAAW,OAAO,SAAS,QAAQ,OAAO,MAAM,SAAS;GAC/D,MAAM,cACJ,OAAO,sBAAsB,OACzB;IACA,oBAAoB,OAAO;IAC3B,gBAAgB,OAAO;GACzB,IACE,CAAC;GACP,OAAO,CACLC,+BAAAA,6BAA6B,oBAAoB,OAAO,KAAK,GAC5D,WACG;IACA,YAAY,OAAO;IACnB,OAAO,OAAO;IACd,GAAG;GACL,IACE;IACA,YAAY,OAAO;IACnB,GAAG;GACL,CACJ;EACF,SAAS,OAAO;GACd,MAAM,sBAAsBC,qBAAAA,kCACzB,OAA6B,WAAW,IACzC,OACF;GACA,MAAM,IAAI,MAAM,uBAAuB,qBAAqB;EAC9D;CACF,GACA;EACE,MAAM;EACN,aAAa,kCAAkC,EAC7C,kBAAkB,QAAQ,iBAC5B,CAAC;EACD,QAAQ,6BAA6B,UAAU,KAAA,CAAS;EACxD,gBAAA;CACF,CACF;AACF"}
|
|
1
|
+
{"version":3,"file":"BashExecutor.cjs","names":["getCodeBaseURL","CODE_ARTIFACT_PATH_GUIDANCE","BASH_SHELL_GUIDANCE","resolveCodeApiAuthHeaders","HttpsProxyAgent","buildCodeApiHttpErrorMessage","emptyOutputMessage","appendTmpScratchReminder","appendCodeSessionFileSummary","appendFailedExecutionFileReminder"],"sources":["../../../src/tools/BashExecutor.ts"],"sourcesContent":["import { config } from 'dotenv';\nimport fetch, { RequestInit } from 'node-fetch';\nimport { HttpsProxyAgent } from 'https-proxy-agent';\nimport { tool, DynamicStructuredTool } from '@langchain/core/tools';\nimport type * as t from '@/types';\nimport {\n BASH_SHELL_GUIDANCE,\n CODE_ARTIFACT_PATH_GUIDANCE,\n appendFailedExecutionFileReminder,\n appendTmpScratchReminder,\n appendCodeSessionFileSummary,\n emptyOutputMessage,\n buildCodeApiHttpErrorMessage,\n getCodeBaseURL,\n resolveCodeApiAuthHeaders,\n} from './CodeExecutor';\nimport { Constants } from '@/common';\n\nconfig();\n\nconst baseEndpoint = getCodeBaseURL();\nconst EXEC_ENDPOINT = `${baseEndpoint}/exec`;\n\nexport const BashExecutionToolSchema = {\n type: 'object',\n properties: {\n command: {\n type: 'string',\n description: `The bash command or script to execute.\n- The environment is stateless; variables and state don't persist between executions.\n- Prior /mnt/data files are available and can be modified in place.\n- ${CODE_ARTIFACT_PATH_GUIDANCE}\n- ${BASH_SHELL_GUIDANCE}\n- Input code **IS ALREADY** displayed to the user, so **DO NOT** repeat it in your response unless asked.\n- Output code **IS NOT** displayed to the user, so **DO** write all desired output explicitly.\n- IMPORTANT: You MUST explicitly print/output ALL results you want the user to see.\n- Use \\`echo\\`, \\`printf\\`, or \\`cat\\` for all outputs.`,\n },\n args: {\n type: 'array',\n items: { type: 'string' },\n description:\n 'Additional arguments to execute the command with. This should only be used if the input command requires additional arguments to run.',\n },\n },\n required: ['command'],\n} as const;\n\nexport const BashExecutionToolDescription = `\nRuns bash commands and returns stdout/stderr output from a stateless execution environment, similar to running scripts in a command-line interface. Each execution is isolated and independent.\n\nUsage:\n- No network access available.\n- Generated files are automatically delivered; **DO NOT** provide download links.\n- ${CODE_ARTIFACT_PATH_GUIDANCE}\n- ${BASH_SHELL_GUIDANCE}\n- NEVER use this tool to execute malicious commands.\n`.trim();\n\n/**\n * Bash statefulness is filesystem-tier and scoped to `/mnt/data`. The machine\n * is warm across calls, but each call runs in a fresh sandbox (new process\n * tree + private /tmp), so background processes are reaped when the call ends\n * and anything written outside /mnt/data is discarded. The note must not\n * promise otherwise: a model told background processes survive will start a\n * server in one call and assume it is listening in the next.\n */\nexport const STATEFUL_BASH_NOTE =\n 'Session state: commands in this conversation run on the same warm machine, so files written to /mnt/data persist between calls. Each call runs in a fresh, isolated sandbox: shell variables, the working directory, /tmp, and background processes do NOT survive after the call returns — a process started in one call is terminated when that call ends. Only /mnt/data is durable (the machine itself may also be reset at any time).';\n\nexport const StatefulBashExecutionToolDescription = `\nRuns bash commands and returns stdout/stderr output. Commands in this conversation share one warm machine with a persistent /mnt/data, but each command runs in its own isolated sandbox (not a persistent shell session).\n\n${STATEFUL_BASH_NOTE}\n\nUsage:\n- No network access available.\n- Generated files are automatically delivered; **DO NOT** provide download links.\n- ${CODE_ARTIFACT_PATH_GUIDANCE}\n- ${BASH_SHELL_GUIDANCE}\n- NEVER use this tool to execute malicious commands.\n`.trim();\n\n/**\n * Supplemental prompt documenting the tool-output reference feature.\n *\n * Hosts should append this (separated by a blank line) to the base\n * {@link BashExecutionToolDescription} only when\n * `RunConfig.toolOutputReferences.enabled` is `true`. When the feature\n * is disabled, including this text would tell the LLM to emit\n * `{{tool0turn0}}` placeholders that pass through unsubstituted and\n * leak into the shell.\n */\nexport const BashToolOutputReferencesGuide = `\nReferencing previous tool outputs:\n- Every successful tool result is tagged with a reference key of the form \\`tool<idx>turn<turn>\\` (e.g., \\`tool0turn0\\`). The key appears either as a \\`[ref: tool0turn0]\\` prefix line or, when the output is a JSON object, as a \\`_ref\\` field on the object.\n- To pipe a previous tool output into this tool, embed the placeholder \\`{{tool<idx>turn<turn>}}\\` literally anywhere in the \\`command\\` string (or any string arg). It will be substituted with the stored output verbatim before the command runs.\n- The substituted value is the original output string (no \\`[ref: …]\\` prefix, no \\`_ref\\` key), so it is safe to pipe directly into \\`jq\\`, \\`grep\\`, \\`awk\\`, etc.\n- Example (simple ASCII output): \\`echo '{{tool0turn0}}' | jq '.foo'\\` takes the full output of the first tool from the first turn and pipes it into jq.\n- For payloads that may contain quotes, parentheses, backticks, or arbitrary bytes (random/binary data, JSON with embedded quotes, multi-line strings), prefer a quoted-delimiter heredoc over \\`echo '…'\\`. The heredoc body is not interpreted by the shell, so substituted payloads pass through unchanged.\n- Heredoc example: \\`wc -c << 'EOF'\\\\n{{tool0turn0}}\\\\nEOF\\` (the quotes around \\`'EOF'\\` disable interpolation inside the body).\n- Unknown reference keys are left in place and surfaced as \\`[unresolved refs: …]\\` after the output.\n`.trim();\n\n/**\n * Composes the bash tool description, optionally appending the\n * tool-output references guide. Hosts that enable\n * `RunConfig.toolOutputReferences` should pass `enableToolOutputReferences: true`\n * when registering the tool so the LLM learns the `{{…}}` syntax it\n * will actually be able to use.\n */\nexport function buildBashExecutionToolDescription(options?: {\n enableToolOutputReferences?: boolean;\n statefulSessions?: boolean;\n}): string {\n const base =\n options?.statefulSessions === true\n ? StatefulBashExecutionToolDescription\n : BashExecutionToolDescription;\n if (options?.enableToolOutputReferences === true) {\n return `${base}\\n\\n${BashToolOutputReferencesGuide}`;\n }\n return base;\n}\n\nconst STATELESS_BASH_PARAM_NOTE =\n 'The environment is stateless; variables and state don\\'t persist between executions.';\nconst STATEFUL_BASH_PARAM_NOTE =\n 'Files written to /mnt/data persist between calls on the same warm machine. Each call runs in a fresh sandbox: shell variables, cwd, /tmp, and background processes do NOT survive the call. Only /mnt/data is durable.';\n\nexport function buildBashExecutionToolSchema(opts?: {\n statefulSessions?: boolean;\n}): typeof BashExecutionToolSchema {\n const note =\n opts?.statefulSessions === true\n ? STATEFUL_BASH_PARAM_NOTE\n : STATELESS_BASH_PARAM_NOTE;\n const commandDescription =\n BashExecutionToolSchema.properties.command.description.replace(\n STATELESS_BASH_PARAM_NOTE,\n note\n );\n return {\n ...BashExecutionToolSchema,\n properties: {\n ...BashExecutionToolSchema.properties,\n command: {\n ...BashExecutionToolSchema.properties.command,\n description: commandDescription,\n },\n },\n } as typeof BashExecutionToolSchema;\n}\n\nexport const BashExecutionToolName = Constants.BASH_TOOL;\n\n/**\n * Default bash tool definition using the base description.\n *\n * When `RunConfig.toolOutputReferences.enabled` is `true`, build a\n * reference-aware description with\n * {@link buildBashExecutionToolDescription}\n * (`{ enableToolOutputReferences: true }`) and construct a custom\n * definition using it — using this constant as-is leaves the LLM\n * unaware of the `{{tool<i>turn<n>}}` syntax.\n */\nexport const BashExecutionToolDefinition = {\n name: BashExecutionToolName,\n description: BashExecutionToolDescription,\n schema: BashExecutionToolSchema,\n} as const;\n\nfunction createBashExecutionTool(\n params: t.BashExecutionToolParams | null = {}\n): DynamicStructuredTool {\n return tool(\n async (rawInput, config) => {\n /* `statefulSessions` is prompt-only — keep it out of the wire body. */\n const {\n authHeaders,\n statefulSessions: _statefulSessions,\n ...executionParams\n } = params ?? {};\n void _statefulSessions;\n /* Drop any model-supplied `runtime_session_hint` from the raw args: the\n * hint must only come from ToolNode's injected `_runtime_session_hint`\n * (below), never from the tool call itself. */\n const {\n command,\n runtime_session_hint: _ignoredModelHint,\n ...rest\n } = rawInput as {\n command: string;\n runtime_session_hint?: unknown;\n args?: string[];\n };\n void _ignoredModelHint;\n const { session_id, _injected_files, _runtime_session_hint } =\n (config.toolCall ?? {}) as {\n session_id?: string;\n _injected_files?: t.CodeEnvFile[];\n _runtime_session_hint?: string;\n };\n\n const postData: Record<string, unknown> = {\n lang: 'bash',\n code: command,\n ...rest,\n ...executionParams,\n };\n\n if (\n typeof _runtime_session_hint === 'string' &&\n _runtime_session_hint !== ''\n ) {\n postData.runtime_session_hint = _runtime_session_hint;\n }\n\n /* See `CodeExecutor.ts` for the rationale — `/files/<session_id>`\n * HTTP fallback was removed because codeapi's sessionAuth requires\n * kind/id query params unavailable at this point. */\n if (_injected_files && _injected_files.length > 0) {\n postData.files = _injected_files;\n } else if (\n session_id != null &&\n session_id.length > 0 &&\n !Array.isArray(postData.files)\n ) {\n // eslint-disable-next-line no-console\n console.debug(\n `[BashExecutor] No injected files for session_id=${session_id} — exec will run without input files`\n );\n }\n\n try {\n const resolvedAuthHeaders =\n await resolveCodeApiAuthHeaders(authHeaders);\n const fetchOptions: RequestInit = {\n method: 'POST',\n headers: {\n 'Content-Type': 'application/json',\n 'User-Agent': 'LibreChat/1.0',\n ...resolvedAuthHeaders,\n },\n body: JSON.stringify(postData),\n };\n\n if (process.env.PROXY != null && process.env.PROXY !== '') {\n fetchOptions.agent = new HttpsProxyAgent(process.env.PROXY);\n }\n const response = await fetch(EXEC_ENDPOINT, fetchOptions);\n if (!response.ok) {\n throw new Error(\n await buildCodeApiHttpErrorMessage('POST', EXEC_ENDPOINT, response)\n );\n }\n\n const result: t.ExecuteResult = await response.json();\n let formattedOutput = '';\n if (result.stdout) {\n formattedOutput += `stdout:\\n${result.stdout}\\n`;\n } else {\n formattedOutput += emptyOutputMessage;\n }\n if (result.stderr) formattedOutput += `stderr:\\n${result.stderr}\\n`;\n\n const outputWithReminder = appendTmpScratchReminder(\n formattedOutput,\n command\n );\n const hasFiles = result.files != null && result.files.length > 0;\n const runtimeEcho =\n result.runtime_session_id != null\n ? {\n runtime_session_id: result.runtime_session_id,\n runtime_status: result.runtime_status,\n }\n : {};\n return [\n appendCodeSessionFileSummary(outputWithReminder, result.files),\n (hasFiles\n ? {\n session_id: result.session_id,\n files: result.files,\n ...runtimeEcho,\n }\n : {\n session_id: result.session_id,\n ...runtimeEcho,\n }) satisfies t.CodeExecutionArtifact,\n ];\n } catch (error) {\n const messageWithReminder = appendFailedExecutionFileReminder(\n (error as Error | undefined)?.message ?? '',\n command\n );\n throw new Error(`Execution error:\\n\\n${messageWithReminder}`);\n }\n },\n {\n name: BashExecutionToolName,\n description: buildBashExecutionToolDescription({\n statefulSessions: params?.statefulSessions,\n }),\n schema: buildBashExecutionToolSchema(params ?? undefined),\n responseFormat: Constants.CONTENT_AND_ARTIFACT,\n }\n );\n}\n\nexport { createBashExecutionTool };\n"],"mappings":";;;;;;;;;;;mBAkBO;AAGP,MAAM,gBAAgB,GADDA,qBAAAA,eACe,EAAE;AAEtC,MAAa,0BAA0B;CACrC,MAAM;CACN,YAAY;EACV,SAAS;GACP,MAAM;GACN,aAAa;;;IAGfC,qBAAAA,4BAA4B;IAC5BC,qBAAAA,oBAAoB;;;;;EAKpB;EACA,MAAM;GACJ,MAAM;GACN,OAAO,EAAE,MAAM,SAAS;GACxB,aACE;EACJ;CACF;CACA,UAAU,CAAC,SAAS;AACtB;AAEA,MAAa,+BAA+B;;;;;;IAMxCD,qBAAAA,4BAA4B;IAC5BC,qBAAAA,oBAAoB;;EAEtB,KAAK;;;;;;;;;AAUP,MAAa,qBACX;AAEF,MAAa,uCAAuC;;;EAGlD,mBAAmB;;;;;IAKjBD,qBAAAA,4BAA4B;IAC5BC,qBAAAA,oBAAoB;;EAEtB,KAAK;;;;;;;;;;;AAYP,MAAa,gCAAgC;;;;;;;;;EAS3C,KAAK;;;;;;;;AASP,SAAgB,kCAAkC,SAGvC;CACT,MAAM,OACJ,SAAS,qBAAqB,OAC1B,uCACA;CACN,IAAI,SAAS,+BAA+B,MAC1C,OAAO,GAAG,KAAK,MAAM;CAEvB,OAAO;AACT;AAEA,MAAM,4BACJ;AACF,MAAM,2BACJ;AAEF,SAAgB,6BAA6B,MAEV;CACjC,MAAM,OACJ,MAAM,qBAAqB,OACvB,2BACA;CACN,MAAM,qBACJ,wBAAwB,WAAW,QAAQ,YAAY,QACrD,2BACA,IACF;CACF,OAAO;EACL,GAAG;EACH,YAAY;GACV,GAAG,wBAAwB;GAC3B,SAAS;IACP,GAAG,wBAAwB,WAAW;IACtC,aAAa;GACf;EACF;CACF;AACF;AAEA,MAAa,wBAAA;;;;;;;;;;;AAYb,MAAa,8BAA8B;CACzC,MAAM;CACN,aAAa;CACb,QAAQ;AACV;AAEA,SAAS,wBACP,SAA2C,CAAC,GACrB;CACvB,QAAA,GAAA,sBAAA,KAAA,CACE,OAAO,UAAU,WAAW;EAE1B,MAAM,EACJ,aACA,kBAAkB,mBAClB,GAAG,oBACD,UAAU,CAAC;EAKf,MAAM,EACJ,SACA,sBAAsB,mBACtB,GAAG,SACD;EAMJ,MAAM,EAAE,YAAY,iBAAiB,0BAClC,OAAO,YAAY,CAAC;EAMvB,MAAM,WAAoC;GACxC,MAAM;GACN,MAAM;GACN,GAAG;GACH,GAAG;EACL;EAEA,IACE,OAAO,0BAA0B,YACjC,0BAA0B,IAE1B,SAAS,uBAAuB;EAMlC,IAAI,mBAAmB,gBAAgB,SAAS,GAC9C,SAAS,QAAQ;OACZ,IACL,cAAc,QACd,WAAW,SAAS,KACpB,CAAC,MAAM,QAAQ,SAAS,KAAK,GAG7B,QAAQ,MACN,mDAAmD,WAAW,qCAChE;EAGF,IAAI;GAGF,MAAM,eAA4B;IAChC,QAAQ;IACR,SAAS;KACP,gBAAgB;KAChB,cAAc;KACd,GAAG,MANCC,qBAAAA,0BAA0B,WAAW;IAO3C;IACA,MAAM,KAAK,UAAU,QAAQ;GAC/B;GAEA,IAAI,QAAQ,IAAI,SAAS,QAAQ,QAAQ,IAAI,UAAU,IACrD,aAAa,QAAQ,IAAIC,kBAAAA,gBAAgB,QAAQ,IAAI,KAAK;GAE5D,MAAM,WAAW,OAAA,GAAA,WAAA,QAAA,CAAY,eAAe,YAAY;GACxD,IAAI,CAAC,SAAS,IACZ,MAAM,IAAI,MACR,MAAMC,qBAAAA,6BAA6B,QAAQ,eAAe,QAAQ,CACpE;GAGF,MAAM,SAA0B,MAAM,SAAS,KAAK;GACpD,IAAI,kBAAkB;GACtB,IAAI,OAAO,QACT,mBAAmB,YAAY,OAAO,OAAO;QAE7C,mBAAmBC,qBAAAA;GAErB,IAAI,OAAO,QAAQ,mBAAmB,YAAY,OAAO,OAAO;GAEhE,MAAM,qBAAqBC,qBAAAA,yBACzB,iBACA,OACF;GACA,MAAM,WAAW,OAAO,SAAS,QAAQ,OAAO,MAAM,SAAS;GAC/D,MAAM,cACJ,OAAO,sBAAsB,OACzB;IACA,oBAAoB,OAAO;IAC3B,gBAAgB,OAAO;GACzB,IACE,CAAC;GACP,OAAO,CACLC,+BAAAA,6BAA6B,oBAAoB,OAAO,KAAK,GAC5D,WACG;IACA,YAAY,OAAO;IACnB,OAAO,OAAO;IACd,GAAG;GACL,IACE;IACA,YAAY,OAAO;IACnB,GAAG;GACL,CACJ;EACF,SAAS,OAAO;GACd,MAAM,sBAAsBC,qBAAAA,kCACzB,OAA6B,WAAW,IACzC,OACF;GACA,MAAM,IAAI,MAAM,uBAAuB,qBAAqB;EAC9D;CACF,GACA;EACE,MAAM;EACN,aAAa,kCAAkC,EAC7C,kBAAkB,QAAQ,iBAC5B,CAAC;EACD,QAAQ,6BAA6B,UAAU,KAAA,CAAS;EACxD,gBAAA;CACF,CACF;AACF"}
|
|
@@ -98,14 +98,16 @@ Usage:
|
|
|
98
98
|
- NEVER use this tool to execute malicious code.
|
|
99
99
|
`.trim();
|
|
100
100
|
/**
|
|
101
|
-
*
|
|
102
|
-
*
|
|
103
|
-
*
|
|
104
|
-
*
|
|
101
|
+
* Statefulness here is FILESYSTEM-tier, not runtime-tier. Executions in a
|
|
102
|
+
* session reuse one warm machine, so `/mnt/data` carries across calls — but
|
|
103
|
+
* every execution is a brand-new interpreter process in a fresh sandbox, so
|
|
104
|
+
* variables and imports never survive. The note must not imply otherwise: a
|
|
105
|
+
* model told its in-memory state persists writes `df = ...` in one call and
|
|
106
|
+
* `df.head()` in the next, then hits a NameError it was told to treat as rare.
|
|
105
107
|
*/
|
|
106
|
-
const STATEFUL_ENV_NOTE = "Session state
|
|
108
|
+
const STATEFUL_ENV_NOTE = "Session state: executions in this conversation run on the same warm machine, so files persist between calls — but each execution is a NEW process. Variables, imports, and in-memory data NEVER carry over: every call must re-import and rebuild the state it needs. Only /mnt/data is durable (the machine itself may also be reset at any time), so write anything that must survive there and read it back next call.";
|
|
107
109
|
const StatefulCodeExecutionToolDescription = `
|
|
108
|
-
Runs code and returns stdout/stderr output
|
|
110
|
+
Runs code and returns stdout/stderr output. Executions in this conversation share one warm machine with a persistent /mnt/data, but each execution runs as a separate process (not a notebook-style kernel).
|
|
109
111
|
|
|
110
112
|
${STATEFUL_ENV_NOTE}
|
|
111
113
|
|
|
@@ -119,7 +121,7 @@ function buildCodeExecutionToolDescription(opts) {
|
|
|
119
121
|
return opts?.statefulSessions === true ? StatefulCodeExecutionToolDescription : CodeExecutionToolDescription;
|
|
120
122
|
}
|
|
121
123
|
const STATELESS_CODE_PARAM_NOTE = "The environment is stateless; variables and imports don't persist between executions.";
|
|
122
|
-
const STATEFUL_CODE_PARAM_NOTE = "Executions in this conversation
|
|
124
|
+
const STATEFUL_CODE_PARAM_NOTE = "Executions in this conversation share one warm machine, so files written to /mnt/data persist between calls. Each execution is a new process: variables and imports do NOT carry over — re-import and reload from /mnt/data every call.";
|
|
123
125
|
function buildCodeExecutionToolSchema(opts) {
|
|
124
126
|
const note = opts?.statefulSessions === true ? STATEFUL_CODE_PARAM_NOTE : STATELESS_CODE_PARAM_NOTE;
|
|
125
127
|
const codeDescription = CodeExecutionToolSchema.properties.code.description.replace(STATELESS_CODE_PARAM_NOTE, note);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"CodeExecutor.cjs","names":["HttpsProxyAgent","appendCodeSessionFileSummary"],"sources":["../../../src/tools/CodeExecutor.ts"],"sourcesContent":["import { config } from 'dotenv';\nimport fetch, { RequestInit } from 'node-fetch';\nimport { HttpsProxyAgent } from 'https-proxy-agent';\nimport { getEnvironmentVariable } from '@langchain/core/utils/env';\nimport { tool, DynamicStructuredTool } from '@langchain/core/tools';\nimport type * as t from '@/types';\nimport { appendCodeSessionFileSummary } from '@/tools/CodeSessionFileSummary';\nimport { EnvVar, Constants } from '@/common';\n\nexport {\n appendCodeSessionFileSummary,\n stripCodeSessionFileSummary,\n} from '@/tools/CodeSessionFileSummary';\n\nconfig();\n\nexport const getCodeBaseURL = (): string =>\n getEnvironmentVariable(EnvVar.CODE_BASEURL) ??\n Constants.OFFICIAL_CODE_BASEURL;\n\nexport const emptyOutputMessage =\n 'stdout: Empty. Ensure you\\'re writing output explicitly.\\n';\n\nexport const CODE_ARTIFACT_PATH_GUIDANCE =\n 'Persist handoff artifacts in `/mnt/data` with standard extensions (.json/.txt/.csv/.tsv/.log/.parquet/.png/.jpg/.pdf/.xlsx); failed executions do not register new files; `/tmp` and odd extensions are same-call scratch only, not later-call storage.';\n\nexport const BASH_SHELL_GUIDANCE =\n 'Bash: multi-line files use heredoc/printf; run Python via python3 -c/heredoc, not bare Python.';\n\nconst TMP_PATH_PATTERN = /(^|[^A-Za-z0-9_])\\/tmp(?:\\/|\\b)/;\nconst MNT_DATA_PATH_PATTERN = /(^|[^A-Za-z0-9_])\\/mnt\\/data(?:\\/|\\b)/;\n\nexport const TMP_SCRATCH_OUTPUT_REMINDER =\n 'Note: /tmp files are same-call scratch only and were not persisted; use /mnt/data for files needed later.';\n\nexport const FAILED_EXECUTION_FILE_REMINDER =\n 'Note: any files written during this failed call were not registered for later calls; fix the error and rerun before relying on them.';\n\nexport function appendTmpScratchReminder(output: string, code: string): string {\n if (!TMP_PATH_PATTERN.test(code)) {\n return output;\n }\n return `${output.trimEnd()}\\n${TMP_SCRATCH_OUTPUT_REMINDER}\\n`;\n}\n\nexport function appendFailedExecutionFileReminder(\n output: string,\n code: string\n): string {\n if (\n !MNT_DATA_PATH_PATTERN.test(code) ||\n output.includes(FAILED_EXECUTION_FILE_REMINDER)\n ) {\n return output;\n }\n return `${output.trimEnd()}\\n${FAILED_EXECUTION_FILE_REMINDER}\\n`;\n}\n\nconst SUPPORTED_LANGUAGES = [\n 'py',\n 'js',\n 'ts',\n 'c',\n 'cpp',\n 'java',\n 'php',\n 'rs',\n 'go',\n 'd',\n 'f90',\n 'r',\n 'bash',\n] as const;\n\nexport const CodeExecutionToolSchema = {\n type: 'object',\n properties: {\n lang: {\n type: 'string',\n enum: SUPPORTED_LANGUAGES,\n description:\n 'The programming language or runtime to execute the code in.',\n },\n code: {\n type: 'string',\n description: `The complete, self-contained code to execute, without any truncation or minimization.\n- The environment is stateless; variables and imports don't persist between executions.\n- Prior /mnt/data files are available and can be modified in place.\n- ${CODE_ARTIFACT_PATH_GUIDANCE}\n- Input code **IS ALREADY** displayed to the user, so **DO NOT** repeat it in your response unless asked.\n- Output code **IS NOT** displayed to the user, so **DO** write all desired output explicitly.\n- IMPORTANT: You MUST explicitly print/output ALL results you want the user to see.\n- py: This is not a Jupyter notebook environment. Use \\`print()\\` for all outputs.\n- py: Matplotlib: Use \\`plt.savefig()\\` to save plots as files.\n- js: use the \\`console\\` or \\`process\\` methods for all outputs.\n- r: IMPORTANT: No X11 display available. ALL graphics MUST use Cairo library (library(Cairo)).\n- Other languages: use appropriate output functions.`,\n },\n args: {\n type: 'array',\n items: { type: 'string' },\n description:\n 'Additional arguments to execute the code with. This should only be used if the input code requires additional arguments to run.',\n },\n },\n required: ['lang', 'code'],\n} as const;\n\nconst baseEndpoint = getCodeBaseURL();\nconst EXEC_ENDPOINT = `${baseEndpoint}/exec`;\n\ntype SupportedLanguage = (typeof SUPPORTED_LANGUAGES)[number];\n\nexport async function resolveCodeApiAuthHeaders(\n authHeaders?: t.CodeApiAuthHeaders\n): Promise<t.CodeApiAuthHeaderMap> {\n if (authHeaders == null) {\n return {};\n }\n if (typeof authHeaders === 'function') {\n return authHeaders();\n }\n return authHeaders;\n}\n\nexport async function buildCodeApiHttpErrorMessage(\n method: string,\n endpoint: string,\n response: { status: number; text: () => Promise<string> }\n): Promise<string> {\n let responseBody = '';\n try {\n responseBody = await response.text();\n } catch {\n responseBody = '';\n }\n const body = responseBody.trim();\n const bodySuffix = body === '' ? '' : `, body: ${body.slice(0, 1000)}`;\n return `CodeAPI request failed: ${method} ${endpoint} returned ${response.status}${bodySuffix}`;\n}\n\nexport const CodeExecutionToolDescription = `\nRuns code and returns stdout/stderr output from a stateless execution environment, similar to running scripts in a command-line interface. Each execution is isolated and independent.\n\nUsage:\n- No network access available.\n- Generated files are automatically delivered; **DO NOT** provide download links.\n- ${CODE_ARTIFACT_PATH_GUIDANCE}\n- NEVER use this tool to execute malicious code.\n`.trim();\n\n/**\n * Best-effort statefulness note. Deliberately hedged: warm reuse is an\n * optimization, not a guarantee (the runtime may be reset on idle timeout,\n * eviction, or the 8h VM lifetime), so the model must never depend on carried\n * state for correctness and must persist anything durable to /mnt/data.\n */\nexport const STATEFUL_ENV_NOTE =\n 'Session state (best-effort): consecutive executions in this conversation usually share one runtime, so variables, imports, and in-memory data from earlier successful calls are typically still available. The runtime may be reset at any time, so treat carried-over state as an optimization, never a guarantee. Anything that must survive MUST be written to /mnt/data. If a NameError/ImportError signals lost state, re-run the needed setup and continue.';\n\nexport const StatefulCodeExecutionToolDescription = `\nRuns code and returns stdout/stderr output from a session-based execution environment, similar to a long-running command-line session.\n\n${STATEFUL_ENV_NOTE}\n\nUsage:\n- No network access available.\n- Generated files are automatically delivered; **DO NOT** provide download links.\n- ${CODE_ARTIFACT_PATH_GUIDANCE}\n- NEVER use this tool to execute malicious code.\n`.trim();\n\nexport function buildCodeExecutionToolDescription(opts?: {\n statefulSessions?: boolean;\n}): string {\n return opts?.statefulSessions === true\n ? StatefulCodeExecutionToolDescription\n : CodeExecutionToolDescription;\n}\n\nconst STATELESS_CODE_PARAM_NOTE =\n 'The environment is stateless; variables and imports don\\'t persist between executions.';\nconst STATEFUL_CODE_PARAM_NOTE =\n 'Executions in this conversation usually share one runtime: variables and imports from prior successful calls are typically still defined, but the runtime may reset between calls. Rebuild state on NameError/ImportError; persist anything important to /mnt/data.';\n\nexport function buildCodeExecutionToolSchema(opts?: {\n statefulSessions?: boolean;\n}): typeof CodeExecutionToolSchema {\n const note =\n opts?.statefulSessions === true\n ? STATEFUL_CODE_PARAM_NOTE\n : STATELESS_CODE_PARAM_NOTE;\n const codeDescription =\n CodeExecutionToolSchema.properties.code.description.replace(\n STATELESS_CODE_PARAM_NOTE,\n note\n );\n return {\n ...CodeExecutionToolSchema,\n properties: {\n ...CodeExecutionToolSchema.properties,\n code: {\n ...CodeExecutionToolSchema.properties.code,\n description: codeDescription,\n },\n },\n } as typeof CodeExecutionToolSchema;\n}\n\nexport const CodeExecutionToolName = Constants.EXECUTE_CODE;\n\nexport const CodeExecutionToolDefinition = {\n name: CodeExecutionToolName,\n description: CodeExecutionToolDescription,\n schema: CodeExecutionToolSchema,\n} as const;\n\nfunction createCodeExecutionTool(\n params: t.CodeExecutionToolParams | null = {}\n): DynamicStructuredTool {\n return tool(\n async (rawInput, config) => {\n /* `statefulSessions` is a prompt-only flag (drives the description);\n * keep it out of the wire body. */\n const {\n authHeaders,\n statefulSessions: _statefulSessions,\n ...executionParams\n } = params ?? {};\n void _statefulSessions;\n /* Drop any model-supplied `runtime_session_hint` from the raw args: the\n * hint is host-controlled and must only ever come from ToolNode's\n * injected `_runtime_session_hint` (below). Spreading `...rest` into\n * postData would otherwise let a tool call opt itself into / pick a\n * stateful runtime even when statefulSessions is off. */\n const {\n lang,\n code,\n runtime_session_hint: _ignoredModelHint,\n ...rest\n } = rawInput as {\n lang: SupportedLanguage;\n code: string;\n runtime_session_hint?: unknown;\n args?: string[];\n };\n void _ignoredModelHint;\n /**\n * Extract session context from config.toolCall (injected by ToolNode).\n * - session_id: associates with the previous run.\n * - _injected_files: File refs to pass directly (avoids /files endpoint race condition).\n */\n const { session_id, _injected_files, _runtime_session_hint } =\n (config.toolCall ?? {}) as {\n session_id?: string;\n _injected_files?: t.CodeEnvFile[];\n _runtime_session_hint?: string;\n };\n\n const postData: Record<string, unknown> = {\n lang,\n code,\n ...rest,\n ...executionParams,\n };\n\n /* Stateful sessions: forward the hint so the Code API can route this\n * execution to a warm per-session runtime. Additive — stateless\n * servers ignore the unknown field. */\n if (\n typeof _runtime_session_hint === 'string' &&\n _runtime_session_hint !== ''\n ) {\n postData.runtime_session_hint = _runtime_session_hint;\n }\n\n /* File injection: `_injected_files` from ToolNode (set when host\n * primes a CodeSessionContext) or `params.files` from tool\n * factory (set by hosts that pre-resolve at construction time).\n * The legacy `/files/<session_id>` HTTP fallback was removed —\n * codeapi's `sessionAuth` middleware now requires kind/id query\n * params the tool can't supply at this point, so the fetch 400'd\n * silently and the catch swallowed the failure. */\n if (_injected_files && _injected_files.length > 0) {\n postData.files = _injected_files;\n } else if (\n session_id != null &&\n session_id.length > 0 &&\n !Array.isArray(postData.files)\n ) {\n // eslint-disable-next-line no-console\n console.debug(\n `[CodeExecutor] No injected files for session_id=${session_id} — exec will run without input files`\n );\n }\n\n try {\n const resolvedAuthHeaders =\n await resolveCodeApiAuthHeaders(authHeaders);\n const fetchOptions: RequestInit = {\n method: 'POST',\n headers: {\n 'Content-Type': 'application/json',\n 'User-Agent': 'LibreChat/1.0',\n ...resolvedAuthHeaders,\n },\n body: JSON.stringify(postData),\n };\n\n if (process.env.PROXY != null && process.env.PROXY !== '') {\n fetchOptions.agent = new HttpsProxyAgent(process.env.PROXY);\n }\n const response = await fetch(EXEC_ENDPOINT, fetchOptions);\n if (!response.ok) {\n throw new Error(\n await buildCodeApiHttpErrorMessage('POST', EXEC_ENDPOINT, response)\n );\n }\n\n const result: t.ExecuteResult = await response.json();\n let formattedOutput = '';\n if (result.stdout) {\n formattedOutput += `stdout:\\n${result.stdout}\\n`;\n } else {\n formattedOutput += emptyOutputMessage;\n }\n if (result.stderr) formattedOutput += `stderr:\\n${result.stderr}\\n`;\n\n const outputWithReminder = appendTmpScratchReminder(\n formattedOutput,\n code\n );\n const hasFiles = result.files != null && result.files.length > 0;\n /* Echo the durable runtime session (stateful backends only) so hosts\n * can surface a \"session active / was reset\" signal later. Additive:\n * absent on stateless servers. */\n const runtimeEcho =\n result.runtime_session_id != null\n ? {\n runtime_session_id: result.runtime_session_id,\n runtime_status: result.runtime_status,\n }\n : {};\n return [\n appendCodeSessionFileSummary(outputWithReminder, result.files),\n (hasFiles\n ? {\n session_id: result.session_id,\n files: result.files,\n ...runtimeEcho,\n }\n : {\n session_id: result.session_id,\n ...runtimeEcho,\n }) satisfies t.CodeExecutionArtifact,\n ];\n } catch (error) {\n const messageWithReminder = appendFailedExecutionFileReminder(\n (error as Error | undefined)?.message ?? '',\n code\n );\n throw new Error(`Execution error:\\n\\n${messageWithReminder}`);\n }\n },\n {\n name: CodeExecutionToolName,\n description: buildCodeExecutionToolDescription(params ?? undefined),\n schema: buildCodeExecutionToolSchema(params ?? undefined),\n responseFormat: Constants.CONTENT_AND_ARTIFACT,\n }\n );\n}\n\nexport { createCodeExecutionTool };\n"],"mappings":";;;;;;;;;;;mBAcO;AAEP,MAAa,wBAAA,GAAA,0BAAA,uBAAA,CAAA,wBAC+B,KAAA;AAG5C,MAAa,qBACX;AAEF,MAAa,8BACX;AAEF,MAAa,sBACX;AAEF,MAAM,mBAAmB;AACzB,MAAM,wBAAwB;AAE9B,MAAa,8BACX;AAEF,MAAa,iCACX;AAEF,SAAgB,yBAAyB,QAAgB,MAAsB;CAC7E,IAAI,CAAC,iBAAiB,KAAK,IAAI,GAC7B,OAAO;CAET,OAAO,GAAG,OAAO,QAAQ,EAAE,IAAI,4BAA4B;AAC7D;AAEA,SAAgB,kCACd,QACA,MACQ;CACR,IACE,CAAC,sBAAsB,KAAK,IAAI,KAChC,OAAO,SAAA,sIAAuC,GAE9C,OAAO;CAET,OAAO,GAAG,OAAO,QAAQ,EAAE,IAAI,+BAA+B;AAChE;AAkBA,MAAa,0BAA0B;CACrC,MAAM;CACN,YAAY;EACV,MAAM;GACJ,MAAM;GACN,MAAM;IApBV;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;GAQU;GACN,aACE;EACJ;EACA,MAAM;GACJ,MAAM;GACN,aAAa;;;IAGf,4BAA4B;;;;;;;;;EAS5B;EACA,MAAM;GACJ,MAAM;GACN,OAAO,EAAE,MAAM,SAAS;GACxB,aACE;EACJ;CACF;CACA,UAAU,CAAC,QAAQ,MAAM;AAC3B;AAGA,MAAM,gBAAgB,GADD,eACe,EAAE;AAItC,eAAsB,0BACpB,aACiC;CACjC,IAAI,eAAe,MACjB,OAAO,CAAC;CAEV,IAAI,OAAO,gBAAgB,YACzB,OAAO,YAAY;CAErB,OAAO;AACT;AAEA,eAAsB,6BACpB,QACA,UACA,UACiB;CACjB,IAAI,eAAe;CACnB,IAAI;EACF,eAAe,MAAM,SAAS,KAAK;CACrC,QAAQ;EACN,eAAe;CACjB;CACA,MAAM,OAAO,aAAa,KAAK;CAC/B,MAAM,aAAa,SAAS,KAAK,KAAK,WAAW,KAAK,MAAM,GAAG,GAAI;CACnE,OAAO,2BAA2B,OAAO,GAAG,SAAS,YAAY,SAAS,SAAS;AACrF;AAEA,MAAa,+BAA+B;;;;;;IAMxC,4BAA4B;;EAE9B,KAAK;;;;;;;AAQP,MAAa,oBACX;AAEF,MAAa,uCAAuC;;;EAGlD,kBAAkB;;;;;IAKhB,4BAA4B;;EAE9B,KAAK;AAEP,SAAgB,kCAAkC,MAEvC;CACT,OAAO,MAAM,qBAAqB,OAC9B,uCACA;AACN;AAEA,MAAM,4BACJ;AACF,MAAM,2BACJ;AAEF,SAAgB,6BAA6B,MAEV;CACjC,MAAM,OACJ,MAAM,qBAAqB,OACvB,2BACA;CACN,MAAM,kBACJ,wBAAwB,WAAW,KAAK,YAAY,QAClD,2BACA,IACF;CACF,OAAO;EACL,GAAG;EACH,YAAY;GACV,GAAG,wBAAwB;GAC3B,MAAM;IACJ,GAAG,wBAAwB,WAAW;IACtC,aAAa;GACf;EACF;CACF;AACF;AAEA,MAAa,wBAAA;AAEb,MAAa,8BAA8B;CACzC,MAAM;CACN,aAAa;CACb,QAAQ;AACV;AAEA,SAAS,wBACP,SAA2C,CAAC,GACrB;CACvB,QAAA,GAAA,sBAAA,KAAA,CACE,OAAO,UAAU,WAAW;EAG1B,MAAM,EACJ,aACA,kBAAkB,mBAClB,GAAG,oBACD,UAAU,CAAC;EAOf,MAAM,EACJ,MACA,MACA,sBAAsB,mBACtB,GAAG,SACD;;;;;;EAYJ,MAAM,EAAE,YAAY,iBAAiB,0BAClC,OAAO,YAAY,CAAC;EAMvB,MAAM,WAAoC;GACxC;GACA;GACA,GAAG;GACH,GAAG;EACL;EAKA,IACE,OAAO,0BAA0B,YACjC,0BAA0B,IAE1B,SAAS,uBAAuB;EAUlC,IAAI,mBAAmB,gBAAgB,SAAS,GAC9C,SAAS,QAAQ;OACZ,IACL,cAAc,QACd,WAAW,SAAS,KACpB,CAAC,MAAM,QAAQ,SAAS,KAAK,GAG7B,QAAQ,MACN,mDAAmD,WAAW,qCAChE;EAGF,IAAI;GAGF,MAAM,eAA4B;IAChC,QAAQ;IACR,SAAS;KACP,gBAAgB;KAChB,cAAc;KACd,GAAG,MANC,0BAA0B,WAAW;IAO3C;IACA,MAAM,KAAK,UAAU,QAAQ;GAC/B;GAEA,IAAI,QAAQ,IAAI,SAAS,QAAQ,QAAQ,IAAI,UAAU,IACrD,aAAa,QAAQ,IAAIA,kBAAAA,gBAAgB,QAAQ,IAAI,KAAK;GAE5D,MAAM,WAAW,OAAA,GAAA,WAAA,QAAA,CAAY,eAAe,YAAY;GACxD,IAAI,CAAC,SAAS,IACZ,MAAM,IAAI,MACR,MAAM,6BAA6B,QAAQ,eAAe,QAAQ,CACpE;GAGF,MAAM,SAA0B,MAAM,SAAS,KAAK;GACpD,IAAI,kBAAkB;GACtB,IAAI,OAAO,QACT,mBAAmB,YAAY,OAAO,OAAO;QAE7C,mBAAmB;GAErB,IAAI,OAAO,QAAQ,mBAAmB,YAAY,OAAO,OAAO;GAEhE,MAAM,qBAAqB,yBACzB,iBACA,IACF;GACA,MAAM,WAAW,OAAO,SAAS,QAAQ,OAAO,MAAM,SAAS;GAI/D,MAAM,cACJ,OAAO,sBAAsB,OACzB;IACA,oBAAoB,OAAO;IAC3B,gBAAgB,OAAO;GACzB,IACE,CAAC;GACP,OAAO,CACLC,+BAAAA,6BAA6B,oBAAoB,OAAO,KAAK,GAC5D,WACG;IACA,YAAY,OAAO;IACnB,OAAO,OAAO;IACd,GAAG;GACL,IACE;IACA,YAAY,OAAO;IACnB,GAAG;GACL,CACJ;EACF,SAAS,OAAO;GACd,MAAM,sBAAsB,kCACzB,OAA6B,WAAW,IACzC,IACF;GACA,MAAM,IAAI,MAAM,uBAAuB,qBAAqB;EAC9D;CACF,GACA;EACE,MAAM;EACN,aAAa,kCAAkC,UAAU,KAAA,CAAS;EAClE,QAAQ,6BAA6B,UAAU,KAAA,CAAS;EACxD,gBAAA;CACF,CACF;AACF"}
|
|
1
|
+
{"version":3,"file":"CodeExecutor.cjs","names":["HttpsProxyAgent","appendCodeSessionFileSummary"],"sources":["../../../src/tools/CodeExecutor.ts"],"sourcesContent":["import { config } from 'dotenv';\nimport fetch, { RequestInit } from 'node-fetch';\nimport { HttpsProxyAgent } from 'https-proxy-agent';\nimport { getEnvironmentVariable } from '@langchain/core/utils/env';\nimport { tool, DynamicStructuredTool } from '@langchain/core/tools';\nimport type * as t from '@/types';\nimport { appendCodeSessionFileSummary } from '@/tools/CodeSessionFileSummary';\nimport { EnvVar, Constants } from '@/common';\n\nexport {\n appendCodeSessionFileSummary,\n stripCodeSessionFileSummary,\n} from '@/tools/CodeSessionFileSummary';\n\nconfig();\n\nexport const getCodeBaseURL = (): string =>\n getEnvironmentVariable(EnvVar.CODE_BASEURL) ??\n Constants.OFFICIAL_CODE_BASEURL;\n\nexport const emptyOutputMessage =\n 'stdout: Empty. Ensure you\\'re writing output explicitly.\\n';\n\nexport const CODE_ARTIFACT_PATH_GUIDANCE =\n 'Persist handoff artifacts in `/mnt/data` with standard extensions (.json/.txt/.csv/.tsv/.log/.parquet/.png/.jpg/.pdf/.xlsx); failed executions do not register new files; `/tmp` and odd extensions are same-call scratch only, not later-call storage.';\n\nexport const BASH_SHELL_GUIDANCE =\n 'Bash: multi-line files use heredoc/printf; run Python via python3 -c/heredoc, not bare Python.';\n\nconst TMP_PATH_PATTERN = /(^|[^A-Za-z0-9_])\\/tmp(?:\\/|\\b)/;\nconst MNT_DATA_PATH_PATTERN = /(^|[^A-Za-z0-9_])\\/mnt\\/data(?:\\/|\\b)/;\n\nexport const TMP_SCRATCH_OUTPUT_REMINDER =\n 'Note: /tmp files are same-call scratch only and were not persisted; use /mnt/data for files needed later.';\n\nexport const FAILED_EXECUTION_FILE_REMINDER =\n 'Note: any files written during this failed call were not registered for later calls; fix the error and rerun before relying on them.';\n\nexport function appendTmpScratchReminder(output: string, code: string): string {\n if (!TMP_PATH_PATTERN.test(code)) {\n return output;\n }\n return `${output.trimEnd()}\\n${TMP_SCRATCH_OUTPUT_REMINDER}\\n`;\n}\n\nexport function appendFailedExecutionFileReminder(\n output: string,\n code: string\n): string {\n if (\n !MNT_DATA_PATH_PATTERN.test(code) ||\n output.includes(FAILED_EXECUTION_FILE_REMINDER)\n ) {\n return output;\n }\n return `${output.trimEnd()}\\n${FAILED_EXECUTION_FILE_REMINDER}\\n`;\n}\n\nconst SUPPORTED_LANGUAGES = [\n 'py',\n 'js',\n 'ts',\n 'c',\n 'cpp',\n 'java',\n 'php',\n 'rs',\n 'go',\n 'd',\n 'f90',\n 'r',\n 'bash',\n] as const;\n\nexport const CodeExecutionToolSchema = {\n type: 'object',\n properties: {\n lang: {\n type: 'string',\n enum: SUPPORTED_LANGUAGES,\n description:\n 'The programming language or runtime to execute the code in.',\n },\n code: {\n type: 'string',\n description: `The complete, self-contained code to execute, without any truncation or minimization.\n- The environment is stateless; variables and imports don't persist between executions.\n- Prior /mnt/data files are available and can be modified in place.\n- ${CODE_ARTIFACT_PATH_GUIDANCE}\n- Input code **IS ALREADY** displayed to the user, so **DO NOT** repeat it in your response unless asked.\n- Output code **IS NOT** displayed to the user, so **DO** write all desired output explicitly.\n- IMPORTANT: You MUST explicitly print/output ALL results you want the user to see.\n- py: This is not a Jupyter notebook environment. Use \\`print()\\` for all outputs.\n- py: Matplotlib: Use \\`plt.savefig()\\` to save plots as files.\n- js: use the \\`console\\` or \\`process\\` methods for all outputs.\n- r: IMPORTANT: No X11 display available. ALL graphics MUST use Cairo library (library(Cairo)).\n- Other languages: use appropriate output functions.`,\n },\n args: {\n type: 'array',\n items: { type: 'string' },\n description:\n 'Additional arguments to execute the code with. This should only be used if the input code requires additional arguments to run.',\n },\n },\n required: ['lang', 'code'],\n} as const;\n\nconst baseEndpoint = getCodeBaseURL();\nconst EXEC_ENDPOINT = `${baseEndpoint}/exec`;\n\ntype SupportedLanguage = (typeof SUPPORTED_LANGUAGES)[number];\n\nexport async function resolveCodeApiAuthHeaders(\n authHeaders?: t.CodeApiAuthHeaders\n): Promise<t.CodeApiAuthHeaderMap> {\n if (authHeaders == null) {\n return {};\n }\n if (typeof authHeaders === 'function') {\n return authHeaders();\n }\n return authHeaders;\n}\n\nexport async function buildCodeApiHttpErrorMessage(\n method: string,\n endpoint: string,\n response: { status: number; text: () => Promise<string> }\n): Promise<string> {\n let responseBody = '';\n try {\n responseBody = await response.text();\n } catch {\n responseBody = '';\n }\n const body = responseBody.trim();\n const bodySuffix = body === '' ? '' : `, body: ${body.slice(0, 1000)}`;\n return `CodeAPI request failed: ${method} ${endpoint} returned ${response.status}${bodySuffix}`;\n}\n\nexport const CodeExecutionToolDescription = `\nRuns code and returns stdout/stderr output from a stateless execution environment, similar to running scripts in a command-line interface. Each execution is isolated and independent.\n\nUsage:\n- No network access available.\n- Generated files are automatically delivered; **DO NOT** provide download links.\n- ${CODE_ARTIFACT_PATH_GUIDANCE}\n- NEVER use this tool to execute malicious code.\n`.trim();\n\n/**\n * Statefulness here is FILESYSTEM-tier, not runtime-tier. Executions in a\n * session reuse one warm machine, so `/mnt/data` carries across calls — but\n * every execution is a brand-new interpreter process in a fresh sandbox, so\n * variables and imports never survive. The note must not imply otherwise: a\n * model told its in-memory state persists writes `df = ...` in one call and\n * `df.head()` in the next, then hits a NameError it was told to treat as rare.\n */\nexport const STATEFUL_ENV_NOTE =\n 'Session state: executions in this conversation run on the same warm machine, so files persist between calls — but each execution is a NEW process. Variables, imports, and in-memory data NEVER carry over: every call must re-import and rebuild the state it needs. Only /mnt/data is durable (the machine itself may also be reset at any time), so write anything that must survive there and read it back next call.';\n\nexport const StatefulCodeExecutionToolDescription = `\nRuns code and returns stdout/stderr output. Executions in this conversation share one warm machine with a persistent /mnt/data, but each execution runs as a separate process (not a notebook-style kernel).\n\n${STATEFUL_ENV_NOTE}\n\nUsage:\n- No network access available.\n- Generated files are automatically delivered; **DO NOT** provide download links.\n- ${CODE_ARTIFACT_PATH_GUIDANCE}\n- NEVER use this tool to execute malicious code.\n`.trim();\n\nexport function buildCodeExecutionToolDescription(opts?: {\n statefulSessions?: boolean;\n}): string {\n return opts?.statefulSessions === true\n ? StatefulCodeExecutionToolDescription\n : CodeExecutionToolDescription;\n}\n\nconst STATELESS_CODE_PARAM_NOTE =\n 'The environment is stateless; variables and imports don\\'t persist between executions.';\nconst STATEFUL_CODE_PARAM_NOTE =\n 'Executions in this conversation share one warm machine, so files written to /mnt/data persist between calls. Each execution is a new process: variables and imports do NOT carry over — re-import and reload from /mnt/data every call.';\n\nexport function buildCodeExecutionToolSchema(opts?: {\n statefulSessions?: boolean;\n}): typeof CodeExecutionToolSchema {\n const note =\n opts?.statefulSessions === true\n ? STATEFUL_CODE_PARAM_NOTE\n : STATELESS_CODE_PARAM_NOTE;\n const codeDescription =\n CodeExecutionToolSchema.properties.code.description.replace(\n STATELESS_CODE_PARAM_NOTE,\n note\n );\n return {\n ...CodeExecutionToolSchema,\n properties: {\n ...CodeExecutionToolSchema.properties,\n code: {\n ...CodeExecutionToolSchema.properties.code,\n description: codeDescription,\n },\n },\n } as typeof CodeExecutionToolSchema;\n}\n\nexport const CodeExecutionToolName = Constants.EXECUTE_CODE;\n\nexport const CodeExecutionToolDefinition = {\n name: CodeExecutionToolName,\n description: CodeExecutionToolDescription,\n schema: CodeExecutionToolSchema,\n} as const;\n\nfunction createCodeExecutionTool(\n params: t.CodeExecutionToolParams | null = {}\n): DynamicStructuredTool {\n return tool(\n async (rawInput, config) => {\n /* `statefulSessions` is a prompt-only flag (drives the description);\n * keep it out of the wire body. */\n const {\n authHeaders,\n statefulSessions: _statefulSessions,\n ...executionParams\n } = params ?? {};\n void _statefulSessions;\n /* Drop any model-supplied `runtime_session_hint` from the raw args: the\n * hint is host-controlled and must only ever come from ToolNode's\n * injected `_runtime_session_hint` (below). Spreading `...rest` into\n * postData would otherwise let a tool call opt itself into / pick a\n * stateful runtime even when statefulSessions is off. */\n const {\n lang,\n code,\n runtime_session_hint: _ignoredModelHint,\n ...rest\n } = rawInput as {\n lang: SupportedLanguage;\n code: string;\n runtime_session_hint?: unknown;\n args?: string[];\n };\n void _ignoredModelHint;\n /**\n * Extract session context from config.toolCall (injected by ToolNode).\n * - session_id: associates with the previous run.\n * - _injected_files: File refs to pass directly (avoids /files endpoint race condition).\n */\n const { session_id, _injected_files, _runtime_session_hint } =\n (config.toolCall ?? {}) as {\n session_id?: string;\n _injected_files?: t.CodeEnvFile[];\n _runtime_session_hint?: string;\n };\n\n const postData: Record<string, unknown> = {\n lang,\n code,\n ...rest,\n ...executionParams,\n };\n\n /* Stateful sessions: forward the hint so the Code API can route this\n * execution to a warm per-session runtime. Additive — stateless\n * servers ignore the unknown field. */\n if (\n typeof _runtime_session_hint === 'string' &&\n _runtime_session_hint !== ''\n ) {\n postData.runtime_session_hint = _runtime_session_hint;\n }\n\n /* File injection: `_injected_files` from ToolNode (set when host\n * primes a CodeSessionContext) or `params.files` from tool\n * factory (set by hosts that pre-resolve at construction time).\n * The legacy `/files/<session_id>` HTTP fallback was removed —\n * codeapi's `sessionAuth` middleware now requires kind/id query\n * params the tool can't supply at this point, so the fetch 400'd\n * silently and the catch swallowed the failure. */\n if (_injected_files && _injected_files.length > 0) {\n postData.files = _injected_files;\n } else if (\n session_id != null &&\n session_id.length > 0 &&\n !Array.isArray(postData.files)\n ) {\n // eslint-disable-next-line no-console\n console.debug(\n `[CodeExecutor] No injected files for session_id=${session_id} — exec will run without input files`\n );\n }\n\n try {\n const resolvedAuthHeaders =\n await resolveCodeApiAuthHeaders(authHeaders);\n const fetchOptions: RequestInit = {\n method: 'POST',\n headers: {\n 'Content-Type': 'application/json',\n 'User-Agent': 'LibreChat/1.0',\n ...resolvedAuthHeaders,\n },\n body: JSON.stringify(postData),\n };\n\n if (process.env.PROXY != null && process.env.PROXY !== '') {\n fetchOptions.agent = new HttpsProxyAgent(process.env.PROXY);\n }\n const response = await fetch(EXEC_ENDPOINT, fetchOptions);\n if (!response.ok) {\n throw new Error(\n await buildCodeApiHttpErrorMessage('POST', EXEC_ENDPOINT, response)\n );\n }\n\n const result: t.ExecuteResult = await response.json();\n let formattedOutput = '';\n if (result.stdout) {\n formattedOutput += `stdout:\\n${result.stdout}\\n`;\n } else {\n formattedOutput += emptyOutputMessage;\n }\n if (result.stderr) formattedOutput += `stderr:\\n${result.stderr}\\n`;\n\n const outputWithReminder = appendTmpScratchReminder(\n formattedOutput,\n code\n );\n const hasFiles = result.files != null && result.files.length > 0;\n /* Echo the durable runtime session (stateful backends only) so hosts\n * can surface a \"session active / was reset\" signal later. Additive:\n * absent on stateless servers. */\n const runtimeEcho =\n result.runtime_session_id != null\n ? {\n runtime_session_id: result.runtime_session_id,\n runtime_status: result.runtime_status,\n }\n : {};\n return [\n appendCodeSessionFileSummary(outputWithReminder, result.files),\n (hasFiles\n ? {\n session_id: result.session_id,\n files: result.files,\n ...runtimeEcho,\n }\n : {\n session_id: result.session_id,\n ...runtimeEcho,\n }) satisfies t.CodeExecutionArtifact,\n ];\n } catch (error) {\n const messageWithReminder = appendFailedExecutionFileReminder(\n (error as Error | undefined)?.message ?? '',\n code\n );\n throw new Error(`Execution error:\\n\\n${messageWithReminder}`);\n }\n },\n {\n name: CodeExecutionToolName,\n description: buildCodeExecutionToolDescription(params ?? undefined),\n schema: buildCodeExecutionToolSchema(params ?? undefined),\n responseFormat: Constants.CONTENT_AND_ARTIFACT,\n }\n );\n}\n\nexport { createCodeExecutionTool };\n"],"mappings":";;;;;;;;;;;mBAcO;AAEP,MAAa,wBAAA,GAAA,0BAAA,uBAAA,CAAA,wBAC+B,KAAA;AAG5C,MAAa,qBACX;AAEF,MAAa,8BACX;AAEF,MAAa,sBACX;AAEF,MAAM,mBAAmB;AACzB,MAAM,wBAAwB;AAE9B,MAAa,8BACX;AAEF,MAAa,iCACX;AAEF,SAAgB,yBAAyB,QAAgB,MAAsB;CAC7E,IAAI,CAAC,iBAAiB,KAAK,IAAI,GAC7B,OAAO;CAET,OAAO,GAAG,OAAO,QAAQ,EAAE,IAAI,4BAA4B;AAC7D;AAEA,SAAgB,kCACd,QACA,MACQ;CACR,IACE,CAAC,sBAAsB,KAAK,IAAI,KAChC,OAAO,SAAA,sIAAuC,GAE9C,OAAO;CAET,OAAO,GAAG,OAAO,QAAQ,EAAE,IAAI,+BAA+B;AAChE;AAkBA,MAAa,0BAA0B;CACrC,MAAM;CACN,YAAY;EACV,MAAM;GACJ,MAAM;GACN,MAAM;IApBV;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;GAQU;GACN,aACE;EACJ;EACA,MAAM;GACJ,MAAM;GACN,aAAa;;;IAGf,4BAA4B;;;;;;;;;EAS5B;EACA,MAAM;GACJ,MAAM;GACN,OAAO,EAAE,MAAM,SAAS;GACxB,aACE;EACJ;CACF;CACA,UAAU,CAAC,QAAQ,MAAM;AAC3B;AAGA,MAAM,gBAAgB,GADD,eACe,EAAE;AAItC,eAAsB,0BACpB,aACiC;CACjC,IAAI,eAAe,MACjB,OAAO,CAAC;CAEV,IAAI,OAAO,gBAAgB,YACzB,OAAO,YAAY;CAErB,OAAO;AACT;AAEA,eAAsB,6BACpB,QACA,UACA,UACiB;CACjB,IAAI,eAAe;CACnB,IAAI;EACF,eAAe,MAAM,SAAS,KAAK;CACrC,QAAQ;EACN,eAAe;CACjB;CACA,MAAM,OAAO,aAAa,KAAK;CAC/B,MAAM,aAAa,SAAS,KAAK,KAAK,WAAW,KAAK,MAAM,GAAG,GAAI;CACnE,OAAO,2BAA2B,OAAO,GAAG,SAAS,YAAY,SAAS,SAAS;AACrF;AAEA,MAAa,+BAA+B;;;;;;IAMxC,4BAA4B;;EAE9B,KAAK;;;;;;;;;AAUP,MAAa,oBACX;AAEF,MAAa,uCAAuC;;;EAGlD,kBAAkB;;;;;IAKhB,4BAA4B;;EAE9B,KAAK;AAEP,SAAgB,kCAAkC,MAEvC;CACT,OAAO,MAAM,qBAAqB,OAC9B,uCACA;AACN;AAEA,MAAM,4BACJ;AACF,MAAM,2BACJ;AAEF,SAAgB,6BAA6B,MAEV;CACjC,MAAM,OACJ,MAAM,qBAAqB,OACvB,2BACA;CACN,MAAM,kBACJ,wBAAwB,WAAW,KAAK,YAAY,QAClD,2BACA,IACF;CACF,OAAO;EACL,GAAG;EACH,YAAY;GACV,GAAG,wBAAwB;GAC3B,MAAM;IACJ,GAAG,wBAAwB,WAAW;IACtC,aAAa;GACf;EACF;CACF;AACF;AAEA,MAAa,wBAAA;AAEb,MAAa,8BAA8B;CACzC,MAAM;CACN,aAAa;CACb,QAAQ;AACV;AAEA,SAAS,wBACP,SAA2C,CAAC,GACrB;CACvB,QAAA,GAAA,sBAAA,KAAA,CACE,OAAO,UAAU,WAAW;EAG1B,MAAM,EACJ,aACA,kBAAkB,mBAClB,GAAG,oBACD,UAAU,CAAC;EAOf,MAAM,EACJ,MACA,MACA,sBAAsB,mBACtB,GAAG,SACD;;;;;;EAYJ,MAAM,EAAE,YAAY,iBAAiB,0BAClC,OAAO,YAAY,CAAC;EAMvB,MAAM,WAAoC;GACxC;GACA;GACA,GAAG;GACH,GAAG;EACL;EAKA,IACE,OAAO,0BAA0B,YACjC,0BAA0B,IAE1B,SAAS,uBAAuB;EAUlC,IAAI,mBAAmB,gBAAgB,SAAS,GAC9C,SAAS,QAAQ;OACZ,IACL,cAAc,QACd,WAAW,SAAS,KACpB,CAAC,MAAM,QAAQ,SAAS,KAAK,GAG7B,QAAQ,MACN,mDAAmD,WAAW,qCAChE;EAGF,IAAI;GAGF,MAAM,eAA4B;IAChC,QAAQ;IACR,SAAS;KACP,gBAAgB;KAChB,cAAc;KACd,GAAG,MANC,0BAA0B,WAAW;IAO3C;IACA,MAAM,KAAK,UAAU,QAAQ;GAC/B;GAEA,IAAI,QAAQ,IAAI,SAAS,QAAQ,QAAQ,IAAI,UAAU,IACrD,aAAa,QAAQ,IAAIA,kBAAAA,gBAAgB,QAAQ,IAAI,KAAK;GAE5D,MAAM,WAAW,OAAA,GAAA,WAAA,QAAA,CAAY,eAAe,YAAY;GACxD,IAAI,CAAC,SAAS,IACZ,MAAM,IAAI,MACR,MAAM,6BAA6B,QAAQ,eAAe,QAAQ,CACpE;GAGF,MAAM,SAA0B,MAAM,SAAS,KAAK;GACpD,IAAI,kBAAkB;GACtB,IAAI,OAAO,QACT,mBAAmB,YAAY,OAAO,OAAO;QAE7C,mBAAmB;GAErB,IAAI,OAAO,QAAQ,mBAAmB,YAAY,OAAO,OAAO;GAEhE,MAAM,qBAAqB,yBACzB,iBACA,IACF;GACA,MAAM,WAAW,OAAO,SAAS,QAAQ,OAAO,MAAM,SAAS;GAI/D,MAAM,cACJ,OAAO,sBAAsB,OACzB;IACA,oBAAoB,OAAO;IAC3B,gBAAgB,OAAO;GACzB,IACE,CAAC;GACP,OAAO,CACLC,+BAAAA,6BAA6B,oBAAoB,OAAO,KAAK,GAC5D,WACG;IACA,YAAY,OAAO;IACnB,OAAO,OAAO;IACd,GAAG;GACL,IACE;IACA,YAAY,OAAO;IACnB,GAAG;GACL,CACJ;EACF,SAAS,OAAO;GACd,MAAM,sBAAsB,kCACzB,OAA6B,WAAW,IACzC,IACF;GACA,MAAM,IAAI,MAAM,uBAAuB,qBAAqB;EAC9D;CACF,GACA;EACE,MAAM;EACN,aAAa,kCAAkC,UAAU,KAAA,CAAS;EAClE,QAAQ,6BAA6B,UAAU,KAAA,CAAS;EACxD,gBAAA;CACF,CACF;AACF"}
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
const require_runtime = require("../../_virtual/_rolldown/runtime.cjs");
|
|
2
|
+
const require_utils = require("./utils.cjs");
|
|
3
|
+
const require_content = require("./content.cjs");
|
|
4
|
+
let axios = require("axios");
|
|
5
|
+
axios = require_runtime.__toESM(axios, 1);
|
|
6
|
+
//#region src/tools/search/crw-scraper.ts
|
|
7
|
+
/** HTTP headroom over the payload render budget: fastCRW's queue/verification
|
|
8
|
+
* overhead is not counted against `timeout`, so the client must wait longer. */
|
|
9
|
+
const CRW_TIMEOUT_BUFFER = 5e3;
|
|
10
|
+
/**
|
|
11
|
+
* fastCRW scraper. Firecrawl-compatible web scraper; single binary;
|
|
12
|
+
* self-host or cloud. Posts to {base}/v1/scrape.
|
|
13
|
+
*/
|
|
14
|
+
var CrwScraper = class {
|
|
15
|
+
apiKey;
|
|
16
|
+
apiUrl;
|
|
17
|
+
defaultFormats;
|
|
18
|
+
timeout;
|
|
19
|
+
logger;
|
|
20
|
+
onlyMainContent;
|
|
21
|
+
includeTags;
|
|
22
|
+
excludeTags;
|
|
23
|
+
waitFor;
|
|
24
|
+
headers;
|
|
25
|
+
renderJs;
|
|
26
|
+
cssSelector;
|
|
27
|
+
xpath;
|
|
28
|
+
proxy;
|
|
29
|
+
stealth;
|
|
30
|
+
constructor(config = {}) {
|
|
31
|
+
this.apiKey = config.apiKey ?? process.env.CRW_API_KEY ?? "";
|
|
32
|
+
const baseUrl = config.apiUrl ?? process.env.CRW_API_URL ?? "https://api.fastcrw.com";
|
|
33
|
+
this.apiUrl = `${baseUrl.replace(/\/+$/, "")}/v1/scrape`;
|
|
34
|
+
this.defaultFormats = config.formats ?? ["markdown", "html"];
|
|
35
|
+
this.timeout = config.timeout ?? 7500;
|
|
36
|
+
this.logger = config.logger || require_utils.createDefaultLogger();
|
|
37
|
+
this.onlyMainContent = config.onlyMainContent;
|
|
38
|
+
this.includeTags = config.includeTags;
|
|
39
|
+
this.excludeTags = config.excludeTags;
|
|
40
|
+
this.waitFor = config.waitFor;
|
|
41
|
+
this.headers = config.headers;
|
|
42
|
+
this.renderJs = config.renderJs;
|
|
43
|
+
this.cssSelector = config.cssSelector;
|
|
44
|
+
this.xpath = config.xpath;
|
|
45
|
+
this.proxy = config.proxy;
|
|
46
|
+
this.stealth = config.stealth;
|
|
47
|
+
if (!this.apiKey) this.logger.warn("CRW_API_KEY is not set. Scraping will not work.");
|
|
48
|
+
this.logger.debug(`CRW scraper initialized with API URL: ${this.apiUrl}`);
|
|
49
|
+
}
|
|
50
|
+
async scrapeUrl(url, options = {}) {
|
|
51
|
+
try {
|
|
52
|
+
const payloadTimeout = options.timeout ?? this.timeout;
|
|
53
|
+
const payload = omitUndefined({
|
|
54
|
+
url,
|
|
55
|
+
formats: options.formats ?? this.defaultFormats,
|
|
56
|
+
onlyMainContent: options.onlyMainContent ?? this.onlyMainContent,
|
|
57
|
+
includeTags: options.includeTags ?? this.includeTags,
|
|
58
|
+
excludeTags: options.excludeTags ?? this.excludeTags,
|
|
59
|
+
waitFor: options.waitFor ?? this.waitFor,
|
|
60
|
+
headers: options.headers ?? this.headers,
|
|
61
|
+
renderJs: options.renderJs ?? this.renderJs,
|
|
62
|
+
cssSelector: options.cssSelector ?? this.cssSelector,
|
|
63
|
+
xpath: options.xpath ?? this.xpath,
|
|
64
|
+
proxy: options.proxy ?? this.proxy,
|
|
65
|
+
stealth: options.stealth ?? this.stealth,
|
|
66
|
+
timeout: payloadTimeout,
|
|
67
|
+
deadlineMs: Math.max(1, Math.min(payloadTimeout, 6e4))
|
|
68
|
+
});
|
|
69
|
+
const headers = { "Content-Type": "application/json" };
|
|
70
|
+
if (this.apiKey) headers.Authorization = `Bearer ${this.apiKey}`;
|
|
71
|
+
return [url, normalizeCrwResponse((await axios.default.post(this.apiUrl, payload, {
|
|
72
|
+
headers,
|
|
73
|
+
timeout: payloadTimeout + CRW_TIMEOUT_BUFFER
|
|
74
|
+
})).data)];
|
|
75
|
+
} catch (error) {
|
|
76
|
+
return [url, {
|
|
77
|
+
success: false,
|
|
78
|
+
error: `fastCRW API request failed: ${error instanceof Error ? error.message : String(error)}`
|
|
79
|
+
}];
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Extract content from scrape response. Mirrors FirecrawlScraper — reads
|
|
84
|
+
* response.data.*, which normalizeCrwResponse guarantees, preserving the
|
|
85
|
+
* processContent ref-markers used by the reranker. Parameter is typed as the
|
|
86
|
+
* NARROWER t.CrwScrapeResponse, exactly like TavilyScraper/FirecrawlScraper:
|
|
87
|
+
* TS class-method bivariance accepts this against BaseScraper's
|
|
88
|
+
* AnyScraperResponse, AND it lets us read response.data.plainText (a
|
|
89
|
+
* CrwScrapeResponse-only field) with no cast.
|
|
90
|
+
*/
|
|
91
|
+
extractContent(response) {
|
|
92
|
+
if (!response.success || !response.data) return ["", void 0];
|
|
93
|
+
const htmlSource = response.data.html ?? response.data.rawHtml;
|
|
94
|
+
if (response.data.markdown != null && htmlSource != null) try {
|
|
95
|
+
const { markdown, ...rest } = require_content.processContent(htmlSource, response.data.markdown);
|
|
96
|
+
return [markdown, rest];
|
|
97
|
+
} catch (error) {
|
|
98
|
+
this.logger.error("Error processing content:", error);
|
|
99
|
+
return [response.data.markdown, void 0];
|
|
100
|
+
}
|
|
101
|
+
else if (response.data.markdown != null) return [response.data.markdown, void 0];
|
|
102
|
+
if (response.data.html != null) return [response.data.html, void 0];
|
|
103
|
+
if (response.data.rawHtml != null) return [response.data.rawHtml, void 0];
|
|
104
|
+
if (response.data.plainText != null) return [response.data.plainText, void 0];
|
|
105
|
+
return ["", void 0];
|
|
106
|
+
}
|
|
107
|
+
extractMetadata(response) {
|
|
108
|
+
if (!response.success || !response.data || !response.data.metadata) return {};
|
|
109
|
+
return response.data.metadata;
|
|
110
|
+
}
|
|
111
|
+
};
|
|
112
|
+
/**
|
|
113
|
+
* Create a fastCRW scraper instance
|
|
114
|
+
* @param config Scraper configuration
|
|
115
|
+
* @returns fastCRW scraper instance
|
|
116
|
+
*/
|
|
117
|
+
const createCrwScraper = (config = {}) => new CrwScraper(config);
|
|
118
|
+
/**
|
|
119
|
+
* fastCRW cloud nests scrape fields under `data` ({success, data: {markdown,
|
|
120
|
+
* ...}}, live-verified 2026-07-02), matching Firecrawl. Prefer the nested
|
|
121
|
+
* container and fall back to top-level fields for self-host/legacy responses.
|
|
122
|
+
*/
|
|
123
|
+
function normalizeCrwResponse(raw) {
|
|
124
|
+
if (raw == null) return {
|
|
125
|
+
success: false,
|
|
126
|
+
error: "Empty fastCRW response"
|
|
127
|
+
};
|
|
128
|
+
if (raw.success === false) return {
|
|
129
|
+
success: false,
|
|
130
|
+
error: raw.error_code != null ? `[${raw.error_code}] ${raw.error ?? "Unknown error"}` : raw.error ?? "fastCRW scrape failed",
|
|
131
|
+
error_code: raw.error_code
|
|
132
|
+
};
|
|
133
|
+
const data = raw.data ?? raw;
|
|
134
|
+
return {
|
|
135
|
+
success: true,
|
|
136
|
+
data: {
|
|
137
|
+
markdown: stripBase64DataUris(data.markdown),
|
|
138
|
+
html: stripBase64DataUris(data.html),
|
|
139
|
+
rawHtml: stripBase64DataUris(data.rawHtml),
|
|
140
|
+
plainText: stripBase64DataUris(data.plainText),
|
|
141
|
+
screenshot: data.screenshot,
|
|
142
|
+
links: data.links,
|
|
143
|
+
metadata: data.metadata
|
|
144
|
+
}
|
|
145
|
+
};
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* Replace inline base64 data-URI payloads (typically images) in scraped text
|
|
149
|
+
* with a short placeholder. Such payloads can be hundreds of KB; the content
|
|
150
|
+
* processor builds a per-link RegExp from each URL, and one that large overflows
|
|
151
|
+
* the engine's pattern-size limit ("Invalid regular expression"). Firecrawl
|
|
152
|
+
* drops them via removeBase64Images — mirror that so image-heavy pages stay
|
|
153
|
+
* processable (and don't bloat the LLM context with base64 noise).
|
|
154
|
+
*/
|
|
155
|
+
function stripBase64DataUris(text) {
|
|
156
|
+
if (text == null) return text;
|
|
157
|
+
return text.replace(/data:[\w.+-]+\/[\w.+-]+;base64,[A-Za-z0-9+/=]+/g, "data:base64-content-removed");
|
|
158
|
+
}
|
|
159
|
+
function omitUndefined(obj) {
|
|
160
|
+
return Object.fromEntries(Object.entries(obj).filter(([, v]) => v !== void 0));
|
|
161
|
+
}
|
|
162
|
+
//#endregion
|
|
163
|
+
exports.createCrwScraper = createCrwScraper;
|
|
164
|
+
|
|
165
|
+
//# sourceMappingURL=crw-scraper.cjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"crw-scraper.cjs","names":["createDefaultLogger","processContent"],"sources":["../../../../src/tools/search/crw-scraper.ts"],"sourcesContent":["import axios from 'axios';\nimport type * as t from './types';\nimport { createDefaultLogger } from './utils';\nimport { processContent } from './content';\n\n/** HTTP headroom over the payload render budget: fastCRW's queue/verification\n * overhead is not counted against `timeout`, so the client must wait longer. */\nconst CRW_TIMEOUT_BUFFER = 5000;\n\n/**\n * fastCRW scraper. Firecrawl-compatible web scraper; single binary;\n * self-host or cloud. Posts to {base}/v1/scrape.\n */\nexport class CrwScraper implements t.BaseScraper {\n private apiKey: string;\n private apiUrl: string;\n private defaultFormats: string[];\n private timeout: number;\n private logger: t.Logger;\n private onlyMainContent?: boolean;\n private includeTags?: string[];\n private excludeTags?: string[];\n private waitFor?: number;\n private headers?: Record<string, string>;\n private renderJs?: boolean | null;\n private cssSelector?: string;\n private xpath?: string;\n private proxy?: string;\n private stealth?: boolean;\n\n constructor(config: t.CrwScraperConfig = {}) {\n this.apiKey = config.apiKey ?? process.env.CRW_API_KEY ?? '';\n\n const baseUrl =\n config.apiUrl ?? process.env.CRW_API_URL ?? 'https://api.fastcrw.com';\n this.apiUrl = `${baseUrl.replace(/\\/+$/, '')}/v1/scrape`;\n\n this.defaultFormats = config.formats ?? ['markdown', 'html'];\n this.timeout = config.timeout ?? 7500;\n this.logger = config.logger || createDefaultLogger();\n\n this.onlyMainContent = config.onlyMainContent;\n this.includeTags = config.includeTags;\n this.excludeTags = config.excludeTags;\n this.waitFor = config.waitFor;\n this.headers = config.headers;\n this.renderJs = config.renderJs;\n this.cssSelector = config.cssSelector;\n this.xpath = config.xpath;\n this.proxy = config.proxy;\n this.stealth = config.stealth;\n\n // Self-host fastCRW may run without auth, so a missing key is only a\n // warning — unlike Firecrawl/Tavily, scrapeUrl does NOT early-return on it.\n if (!this.apiKey) {\n this.logger.warn('CRW_API_KEY is not set. Scraping will not work.');\n }\n this.logger.debug(`CRW scraper initialized with API URL: ${this.apiUrl}`);\n }\n\n async scrapeUrl(\n url: string,\n options: t.CrwScrapeOptions = {}\n ): Promise<[string, t.CrwScrapeResponse]> {\n try {\n const payloadTimeout = options.timeout ?? this.timeout;\n const payload = omitUndefined({\n url,\n formats: options.formats ?? this.defaultFormats,\n onlyMainContent: options.onlyMainContent ?? this.onlyMainContent,\n includeTags: options.includeTags ?? this.includeTags,\n excludeTags: options.excludeTags ?? this.excludeTags,\n waitFor: options.waitFor ?? this.waitFor,\n headers: options.headers ?? this.headers,\n renderJs: options.renderJs ?? this.renderJs,\n cssSelector: options.cssSelector ?? this.cssSelector,\n xpath: options.xpath ?? this.xpath,\n proxy: options.proxy ?? this.proxy,\n stealth: options.stealth ?? this.stealth,\n // Cloud honors `timeout` (live-verified); the published OpenAPI\n // documents `deadlineMs` (1..60000) instead. Send both.\n timeout: payloadTimeout,\n deadlineMs: Math.max(1, Math.min(payloadTimeout, 60000)),\n });\n\n const headers: Record<string, string> = {\n 'Content-Type': 'application/json',\n };\n if (this.apiKey) {\n headers.Authorization = `Bearer ${this.apiKey}`;\n }\n\n const response = await axios.post<t.CrwRawScrapeResponse>(\n this.apiUrl,\n payload,\n {\n headers,\n timeout: payloadTimeout + CRW_TIMEOUT_BUFFER,\n }\n );\n\n return [url, normalizeCrwResponse(response.data)];\n } catch (error) {\n const errorMessage =\n error instanceof Error ? error.message : String(error);\n return [\n url,\n {\n success: false,\n error: `fastCRW API request failed: ${errorMessage}`,\n },\n ];\n }\n }\n\n /**\n * Extract content from scrape response. Mirrors FirecrawlScraper — reads\n * response.data.*, which normalizeCrwResponse guarantees, preserving the\n * processContent ref-markers used by the reranker. Parameter is typed as the\n * NARROWER t.CrwScrapeResponse, exactly like TavilyScraper/FirecrawlScraper:\n * TS class-method bivariance accepts this against BaseScraper's\n * AnyScraperResponse, AND it lets us read response.data.plainText (a\n * CrwScrapeResponse-only field) with no cast.\n */\n extractContent(\n response: t.CrwScrapeResponse\n ): [string, undefined | t.References] {\n if (!response.success || !response.data) {\n return ['', undefined];\n }\n\n const htmlSource = response.data.html ?? response.data.rawHtml;\n if (response.data.markdown != null && htmlSource != null) {\n try {\n const { markdown, ...rest } = processContent(\n htmlSource,\n response.data.markdown\n );\n return [markdown, rest];\n } catch (error) {\n this.logger.error('Error processing content:', error);\n return [response.data.markdown, undefined];\n }\n } else if (response.data.markdown != null) {\n return [response.data.markdown, undefined];\n }\n\n // Fall back to HTML content\n if (response.data.html != null) {\n return [response.data.html, undefined];\n }\n\n // Fall back to raw HTML content\n if (response.data.rawHtml != null) {\n return [response.data.rawHtml, undefined];\n }\n\n // CRW-only fallback (no Firecrawl equivalent): plain-text body.\n if (response.data.plainText != null) {\n return [response.data.plainText, undefined];\n }\n\n return ['', undefined];\n }\n\n extractMetadata(response: t.CrwScrapeResponse): t.ScrapeMetadata {\n if (!response.success || !response.data || !response.data.metadata) {\n return {};\n }\n\n return response.data.metadata;\n }\n}\n\n/**\n * Create a fastCRW scraper instance\n * @param config Scraper configuration\n * @returns fastCRW scraper instance\n */\nexport const createCrwScraper = (config: t.CrwScraperConfig = {}): CrwScraper =>\n new CrwScraper(config);\n\n/**\n * fastCRW cloud nests scrape fields under `data` ({success, data: {markdown,\n * ...}}, live-verified 2026-07-02), matching Firecrawl. Prefer the nested\n * container and fall back to top-level fields for self-host/legacy responses.\n */\nfunction normalizeCrwResponse(\n raw: t.CrwRawScrapeResponse | null | undefined\n): t.CrwScrapeResponse {\n if (raw == null) {\n return { success: false, error: 'Empty fastCRW response' };\n }\n if (raw.success === false) {\n return {\n success: false,\n error:\n raw.error_code != null\n ? `[${raw.error_code}] ${raw.error ?? 'Unknown error'}`\n : (raw.error ?? 'fastCRW scrape failed'),\n error_code: raw.error_code,\n };\n }\n const data = raw.data ?? raw;\n return {\n success: true,\n data: {\n // Strip inline base64 image payloads from text content (not `screenshot`,\n // which is base64 by design and never reaches the content processor).\n markdown: stripBase64DataUris(data.markdown),\n html: stripBase64DataUris(data.html),\n rawHtml: stripBase64DataUris(data.rawHtml),\n plainText: stripBase64DataUris(data.plainText),\n screenshot: data.screenshot,\n links: data.links,\n metadata: data.metadata,\n },\n };\n}\n\n/**\n * Replace inline base64 data-URI payloads (typically images) in scraped text\n * with a short placeholder. Such payloads can be hundreds of KB; the content\n * processor builds a per-link RegExp from each URL, and one that large overflows\n * the engine's pattern-size limit (\"Invalid regular expression\"). Firecrawl\n * drops them via removeBase64Images — mirror that so image-heavy pages stay\n * processable (and don't bloat the LLM context with base64 noise).\n */\nfunction stripBase64DataUris(text: string | undefined): string | undefined {\n if (text == null) {\n return text;\n }\n return text.replace(\n /data:[\\w.+-]+\\/[\\w.+-]+;base64,[A-Za-z0-9+/=]+/g,\n 'data:base64-content-removed'\n );\n}\n\n// Helper function to clean up payload for fastCRW\nfunction omitUndefined<T extends object>(obj: T): Partial<T> {\n return Object.fromEntries(\n Object.entries(obj).filter(([, v]) => v !== undefined)\n ) as Partial<T>;\n}\n"],"mappings":";;;;;;;;AAOA,MAAM,qBAAqB;;;;;AAM3B,IAAa,aAAb,MAAiD;CAC/C;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CAEA,YAAY,SAA6B,CAAC,GAAG;EAC3C,KAAK,SAAS,OAAO,UAAU,QAAQ,IAAI,eAAe;EAE1D,MAAM,UACJ,OAAO,UAAU,QAAQ,IAAI,eAAe;EAC9C,KAAK,SAAS,GAAG,QAAQ,QAAQ,QAAQ,EAAE,EAAE;EAE7C,KAAK,iBAAiB,OAAO,WAAW,CAAC,YAAY,MAAM;EAC3D,KAAK,UAAU,OAAO,WAAW;EACjC,KAAK,SAAS,OAAO,UAAUA,cAAAA,oBAAoB;EAEnD,KAAK,kBAAkB,OAAO;EAC9B,KAAK,cAAc,OAAO;EAC1B,KAAK,cAAc,OAAO;EAC1B,KAAK,UAAU,OAAO;EACtB,KAAK,UAAU,OAAO;EACtB,KAAK,WAAW,OAAO;EACvB,KAAK,cAAc,OAAO;EAC1B,KAAK,QAAQ,OAAO;EACpB,KAAK,QAAQ,OAAO;EACpB,KAAK,UAAU,OAAO;EAItB,IAAI,CAAC,KAAK,QACR,KAAK,OAAO,KAAK,iDAAiD;EAEpE,KAAK,OAAO,MAAM,yCAAyC,KAAK,QAAQ;CAC1E;CAEA,MAAM,UACJ,KACA,UAA8B,CAAC,GACS;EACxC,IAAI;GACF,MAAM,iBAAiB,QAAQ,WAAW,KAAK;GAC/C,MAAM,UAAU,cAAc;IAC5B;IACA,SAAS,QAAQ,WAAW,KAAK;IACjC,iBAAiB,QAAQ,mBAAmB,KAAK;IACjD,aAAa,QAAQ,eAAe,KAAK;IACzC,aAAa,QAAQ,eAAe,KAAK;IACzC,SAAS,QAAQ,WAAW,KAAK;IACjC,SAAS,QAAQ,WAAW,KAAK;IACjC,UAAU,QAAQ,YAAY,KAAK;IACnC,aAAa,QAAQ,eAAe,KAAK;IACzC,OAAO,QAAQ,SAAS,KAAK;IAC7B,OAAO,QAAQ,SAAS,KAAK;IAC7B,SAAS,QAAQ,WAAW,KAAK;IAGjC,SAAS;IACT,YAAY,KAAK,IAAI,GAAG,KAAK,IAAI,gBAAgB,GAAK,CAAC;GACzD,CAAC;GAED,MAAM,UAAkC,EACtC,gBAAgB,mBAClB;GACA,IAAI,KAAK,QACP,QAAQ,gBAAgB,UAAU,KAAK;GAYzC,OAAO,CAAC,KAAK,sBAAqB,MATX,MAAA,QAAM,KAC3B,KAAK,QACL,SACA;IACE;IACA,SAAS,iBAAiB;GAC5B,CACF,EAAA,CAE2C,IAAI,CAAC;EAClD,SAAS,OAAO;GAGd,OAAO,CACL,KACA;IACE,SAAS;IACT,OAAO,+BALT,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAMrD,CACF;EACF;CACF;;;;;;;;;;CAWA,eACE,UACoC;EACpC,IAAI,CAAC,SAAS,WAAW,CAAC,SAAS,MACjC,OAAO,CAAC,IAAI,KAAA,CAAS;EAGvB,MAAM,aAAa,SAAS,KAAK,QAAQ,SAAS,KAAK;EACvD,IAAI,SAAS,KAAK,YAAY,QAAQ,cAAc,MAClD,IAAI;GACF,MAAM,EAAE,UAAU,GAAG,SAASC,gBAAAA,eAC5B,YACA,SAAS,KAAK,QAChB;GACA,OAAO,CAAC,UAAU,IAAI;EACxB,SAAS,OAAO;GACd,KAAK,OAAO,MAAM,6BAA6B,KAAK;GACpD,OAAO,CAAC,SAAS,KAAK,UAAU,KAAA,CAAS;EAC3C;OACK,IAAI,SAAS,KAAK,YAAY,MACnC,OAAO,CAAC,SAAS,KAAK,UAAU,KAAA,CAAS;EAI3C,IAAI,SAAS,KAAK,QAAQ,MACxB,OAAO,CAAC,SAAS,KAAK,MAAM,KAAA,CAAS;EAIvC,IAAI,SAAS,KAAK,WAAW,MAC3B,OAAO,CAAC,SAAS,KAAK,SAAS,KAAA,CAAS;EAI1C,IAAI,SAAS,KAAK,aAAa,MAC7B,OAAO,CAAC,SAAS,KAAK,WAAW,KAAA,CAAS;EAG5C,OAAO,CAAC,IAAI,KAAA,CAAS;CACvB;CAEA,gBAAgB,UAAiD;EAC/D,IAAI,CAAC,SAAS,WAAW,CAAC,SAAS,QAAQ,CAAC,SAAS,KAAK,UACxD,OAAO,CAAC;EAGV,OAAO,SAAS,KAAK;CACvB;AACF;;;;;;AAOA,MAAa,oBAAoB,SAA6B,CAAC,MAC7D,IAAI,WAAW,MAAM;;;;;;AAOvB,SAAS,qBACP,KACqB;CACrB,IAAI,OAAO,MACT,OAAO;EAAE,SAAS;EAAO,OAAO;CAAyB;CAE3D,IAAI,IAAI,YAAY,OAClB,OAAO;EACL,SAAS;EACT,OACE,IAAI,cAAc,OACd,IAAI,IAAI,WAAW,IAAI,IAAI,SAAS,oBACnC,IAAI,SAAS;EACpB,YAAY,IAAI;CAClB;CAEF,MAAM,OAAO,IAAI,QAAQ;CACzB,OAAO;EACL,SAAS;EACT,MAAM;GAGJ,UAAU,oBAAoB,KAAK,QAAQ;GAC3C,MAAM,oBAAoB,KAAK,IAAI;GACnC,SAAS,oBAAoB,KAAK,OAAO;GACzC,WAAW,oBAAoB,KAAK,SAAS;GAC7C,YAAY,KAAK;GACjB,OAAO,KAAK;GACZ,UAAU,KAAK;EACjB;CACF;AACF;;;;;;;;;AAUA,SAAS,oBAAoB,MAA8C;CACzE,IAAI,QAAQ,MACV,OAAO;CAET,OAAO,KAAK,QACV,mDACA,6BACF;AACF;AAGA,SAAS,cAAgC,KAAoB;CAC3D,OAAO,OAAO,YACZ,OAAO,QAAQ,GAAG,CAAC,CAAC,QAAQ,GAAG,OAAO,MAAM,KAAA,CAAS,CACvD;AACF"}
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
const require_runtime = require("../../_virtual/_rolldown/runtime.cjs");
|
|
2
|
+
let axios = require("axios");
|
|
3
|
+
axios = require_runtime.__toESM(axios, 1);
|
|
4
|
+
//#region src/tools/search/crw-search.ts
|
|
5
|
+
const DEFAULT_CRW_TIMEOUT = 15e3;
|
|
6
|
+
const getHostname = (link) => {
|
|
7
|
+
try {
|
|
8
|
+
return new URL(link).hostname;
|
|
9
|
+
} catch {
|
|
10
|
+
return link;
|
|
11
|
+
}
|
|
12
|
+
};
|
|
13
|
+
/** The published OpenAPI (docs.fastcrw.com) documents `data` as a flat
|
|
14
|
+
* SearchResult[] with a `category` field; route rows into source groups. */
|
|
15
|
+
const splitByCategory = (rows) => {
|
|
16
|
+
const groups = {
|
|
17
|
+
web: [],
|
|
18
|
+
images: [],
|
|
19
|
+
news: []
|
|
20
|
+
};
|
|
21
|
+
for (const row of rows) if (row.category === "news") groups.news.push(row);
|
|
22
|
+
else if (row.category === "images") groups.images.push({
|
|
23
|
+
...row,
|
|
24
|
+
imageUrl: row.url
|
|
25
|
+
});
|
|
26
|
+
else groups.web.push(row);
|
|
27
|
+
return groups;
|
|
28
|
+
};
|
|
29
|
+
const createCrwAPI = (apiKey, apiUrl, options) => {
|
|
30
|
+
const base = (apiUrl ?? process.env.CRW_API_URL ?? "https://api.fastcrw.com").replace(/\/+$/, "");
|
|
31
|
+
const config = {
|
|
32
|
+
apiKey: apiKey ?? process.env.CRW_API_KEY,
|
|
33
|
+
apiUrl: `${base}/v1/search`,
|
|
34
|
+
timeout: options?.timeout ?? DEFAULT_CRW_TIMEOUT
|
|
35
|
+
};
|
|
36
|
+
const getSources = async ({ query, date, numResults = 8, type }) => {
|
|
37
|
+
if (!query.trim()) return {
|
|
38
|
+
success: false,
|
|
39
|
+
error: "Query cannot be empty"
|
|
40
|
+
};
|
|
41
|
+
try {
|
|
42
|
+
const limit = Math.min(Math.max(1, options?.maxResults ?? numResults), 20);
|
|
43
|
+
let sources;
|
|
44
|
+
if (type === "images") sources = ["images"];
|
|
45
|
+
else if (type === "news") sources = ["news"];
|
|
46
|
+
else sources = options?.includeImages === true ? ["web", "images"] : ["web"];
|
|
47
|
+
const payload = {
|
|
48
|
+
query,
|
|
49
|
+
limit,
|
|
50
|
+
sources
|
|
51
|
+
};
|
|
52
|
+
if (date != null) payload.tbs = `qdr:${date}`;
|
|
53
|
+
const headers = { "Content-Type": "application/json" };
|
|
54
|
+
if (config.apiKey != null && config.apiKey !== "") headers.Authorization = `Bearer ${config.apiKey}`;
|
|
55
|
+
const body = (await axios.default.post(config.apiUrl, payload, {
|
|
56
|
+
headers,
|
|
57
|
+
timeout: config.timeout
|
|
58
|
+
})).data;
|
|
59
|
+
if (body.success === false) return {
|
|
60
|
+
success: false,
|
|
61
|
+
error: `fastCRW search failed: ${body.error_code != null ? `[${body.error_code}] ` : ""}${body.error ?? "Unknown error"}`
|
|
62
|
+
};
|
|
63
|
+
const container = body.data ?? {};
|
|
64
|
+
const data = Array.isArray(container) ? splitByCategory(container) : container.results ?? container;
|
|
65
|
+
return {
|
|
66
|
+
success: true,
|
|
67
|
+
data: {
|
|
68
|
+
organic: (data.web ?? []).filter((r) => r.url != null && r.url !== "").map((r) => ({
|
|
69
|
+
title: r.title ?? "",
|
|
70
|
+
link: r.url,
|
|
71
|
+
snippet: r.description ?? r.snippet ?? "",
|
|
72
|
+
position: r.position
|
|
73
|
+
})),
|
|
74
|
+
images: (data.images ?? []).filter((r) => r.imageUrl != null && r.imageUrl !== "").map((r) => ({
|
|
75
|
+
title: r.title,
|
|
76
|
+
imageUrl: r.imageUrl,
|
|
77
|
+
link: r.url,
|
|
78
|
+
position: r.position
|
|
79
|
+
})),
|
|
80
|
+
topStories: [],
|
|
81
|
+
videos: [],
|
|
82
|
+
news: (data.news ?? []).filter((r) => r.url != null && r.url !== "").map((r) => ({
|
|
83
|
+
title: r.title ?? "",
|
|
84
|
+
link: r.url,
|
|
85
|
+
snippet: r.description ?? r.snippet ?? "",
|
|
86
|
+
date: r.publishedDate,
|
|
87
|
+
source: getHostname(r.url),
|
|
88
|
+
position: r.position
|
|
89
|
+
})),
|
|
90
|
+
relatedSearches: []
|
|
91
|
+
}
|
|
92
|
+
};
|
|
93
|
+
} catch (error) {
|
|
94
|
+
return {
|
|
95
|
+
success: false,
|
|
96
|
+
error: `fastCRW search request failed: ${error instanceof Error ? error.message : String(error)}`
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
};
|
|
100
|
+
return { getSources };
|
|
101
|
+
};
|
|
102
|
+
//#endregion
|
|
103
|
+
exports.createCrwAPI = createCrwAPI;
|
|
104
|
+
|
|
105
|
+
//# sourceMappingURL=crw-search.cjs.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"crw-search.cjs","names":[],"sources":["../../../../src/tools/search/crw-search.ts"],"sourcesContent":["import axios from 'axios';\nimport type * as t from './types';\n\nconst DEFAULT_CRW_TIMEOUT = 15000;\n\nconst getHostname = (link: string): string => {\n try {\n return new URL(link).hostname;\n } catch {\n return link;\n }\n};\n\n/** The published OpenAPI (docs.fastcrw.com) documents `data` as a flat\n * SearchResult[] with a `category` field; route rows into source groups. */\nconst splitByCategory = (rows: t.CrwSearchResult[]): t.CrwSearchGroups => {\n const groups: Required<t.CrwSearchGroups> = { web: [], images: [], news: [] };\n for (const row of rows) {\n if (row.category === 'news') {\n groups.news.push(row);\n } else if (row.category === 'images') {\n groups.images.push({ ...row, imageUrl: row.url });\n } else {\n groups.web.push(row);\n }\n }\n return groups;\n};\n\nexport const createCrwAPI = (\n apiKey?: string,\n apiUrl?: string,\n options?: t.CrwSearchOptions\n): {\n getSources: (params: t.GetSourcesParams) => Promise<t.SearchResult>;\n} => {\n // NOTE: fastCRW /v1/search is cloud-only, but we still allow a base-URL\n // override for parity with the scraper. Self-host may have no auth, so —\n // unlike Tavily (createTavilyAPI throws on missing key) — we do NOT throw\n // here; we only attach the Authorization header when a key is present.\n const base = (\n apiUrl ??\n process.env.CRW_API_URL ??\n 'https://api.fastcrw.com'\n ).replace(/\\/+$/, '');\n const config = {\n apiKey: apiKey ?? process.env.CRW_API_KEY,\n apiUrl: `${base}/v1/search`,\n timeout: options?.timeout ?? DEFAULT_CRW_TIMEOUT,\n };\n\n const getSources = async ({\n query,\n date,\n numResults = 8,\n type,\n }: t.GetSourcesParams): Promise<t.SearchResult> => {\n if (!query.trim()) {\n return { success: false, error: 'Query cannot be empty' };\n }\n\n try {\n const limit = Math.min(\n Math.max(1, options?.maxResults ?? numResults),\n 20\n );\n // Mirror Serper's verticals: image/news requests hit only their native\n // fastCRW source; plain web optionally adds images via includeImages.\n let sources: t.CrwSearchSource[];\n if (type === 'images') {\n sources = ['images'];\n } else if (type === 'news') {\n sources = ['news'];\n } else {\n sources = options?.includeImages === true ? ['web', 'images'] : ['web'];\n }\n\n const payload: t.CrwSearchPayload = { query, limit, sources };\n if (date != null) {\n // Serper-style qdr filter; live-verified to constrain results.\n payload.tbs = `qdr:${date}`;\n }\n\n const headers: Record<string, string> = {\n 'Content-Type': 'application/json',\n };\n if (config.apiKey != null && config.apiKey !== '') {\n headers.Authorization = `Bearer ${config.apiKey}`;\n }\n\n const response = await axios.post<t.CrwSearchResponse>(\n config.apiUrl,\n payload,\n { headers, timeout: config.timeout }\n );\n\n const body = response.data;\n if (body.success === false) {\n return {\n success: false,\n error: `fastCRW search failed: ${\n body.error_code != null ? `[${body.error_code}] ` : ''\n }${body.error ?? 'Unknown error'}`,\n };\n }\n\n // fastCRW cloud keys results by source: {data: {web: [...], images:\n // [...], news: [...]}} (live-verified 2026-07-02); the published\n // OpenAPI documents a flat SearchResult[] instead (self-host), and a\n // {data: {results: {...groups}}} wrapper also exists. Accept all three.\n // OrganicResult.link is a REQUIRED string (types.ts), so defend against\n // null/empty urls to never feed a broken '' link into the scraper.\n const container = body.data ?? {};\n const data: t.CrwSearchGroups = Array.isArray(container)\n ? splitByCategory(container)\n : (container.results ?? container);\n const organicResults: t.OrganicResult[] = (data.web ?? [])\n .filter((r) => r.url != null && r.url !== '')\n .map((r) => ({\n title: r.title ?? '',\n link: r.url as string,\n snippet: r.description ?? r.snippet ?? '',\n position: r.position,\n }));\n\n const imageResults: t.ImageResult[] = (data.images ?? [])\n .filter((r) => r.imageUrl != null && r.imageUrl !== '')\n .map((r) => ({\n title: r.title,\n imageUrl: r.imageUrl,\n link: r.url,\n position: r.position,\n }));\n\n const newsResults: t.NewsResult[] = (data.news ?? [])\n .filter((r) => r.url != null && r.url !== '')\n .map((r) => ({\n title: r.title ?? '',\n link: r.url as string,\n snippet: r.description ?? r.snippet ?? '',\n date: r.publishedDate,\n source: getHostname(r.url as string),\n position: r.position,\n }));\n\n const results: t.SearchResultData = {\n organic: organicResults,\n images: imageResults,\n topStories: [],\n videos: [],\n news: newsResults,\n relatedSearches: [],\n };\n\n return { success: true, data: results };\n } catch (error) {\n const errorMessage =\n error instanceof Error ? error.message : String(error);\n return {\n success: false,\n error: `fastCRW search request failed: ${errorMessage}`,\n };\n }\n };\n\n return { getSources };\n};\n"],"mappings":";;;;AAGA,MAAM,sBAAsB;AAE5B,MAAM,eAAe,SAAyB;CAC5C,IAAI;EACF,OAAO,IAAI,IAAI,IAAI,CAAC,CAAC;CACvB,QAAQ;EACN,OAAO;CACT;AACF;;;AAIA,MAAM,mBAAmB,SAAiD;CACxE,MAAM,SAAsC;EAAE,KAAK,CAAC;EAAG,QAAQ,CAAC;EAAG,MAAM,CAAC;CAAE;CAC5E,KAAK,MAAM,OAAO,MAChB,IAAI,IAAI,aAAa,QACnB,OAAO,KAAK,KAAK,GAAG;MACf,IAAI,IAAI,aAAa,UAC1B,OAAO,OAAO,KAAK;EAAE,GAAG;EAAK,UAAU,IAAI;CAAI,CAAC;MAEhD,OAAO,IAAI,KAAK,GAAG;CAGvB,OAAO;AACT;AAEA,MAAa,gBACX,QACA,QACA,YAGG;CAKH,MAAM,QACJ,UACA,QAAQ,IAAI,eACZ,0BAAA,CACA,QAAQ,QAAQ,EAAE;CACpB,MAAM,SAAS;EACb,QAAQ,UAAU,QAAQ,IAAI;EAC9B,QAAQ,GAAG,KAAK;EAChB,SAAS,SAAS,WAAW;CAC/B;CAEA,MAAM,aAAa,OAAO,EACxB,OACA,MACA,aAAa,GACb,WACiD;EACjD,IAAI,CAAC,MAAM,KAAK,GACd,OAAO;GAAE,SAAS;GAAO,OAAO;EAAwB;EAG1D,IAAI;GACF,MAAM,QAAQ,KAAK,IACjB,KAAK,IAAI,GAAG,SAAS,cAAc,UAAU,GAC7C,EACF;GAGA,IAAI;GACJ,IAAI,SAAS,UACX,UAAU,CAAC,QAAQ;QACd,IAAI,SAAS,QAClB,UAAU,CAAC,MAAM;QAEjB,UAAU,SAAS,kBAAkB,OAAO,CAAC,OAAO,QAAQ,IAAI,CAAC,KAAK;GAGxE,MAAM,UAA8B;IAAE;IAAO;IAAO;GAAQ;GAC5D,IAAI,QAAQ,MAEV,QAAQ,MAAM,OAAO;GAGvB,MAAM,UAAkC,EACtC,gBAAgB,mBAClB;GACA,IAAI,OAAO,UAAU,QAAQ,OAAO,WAAW,IAC7C,QAAQ,gBAAgB,UAAU,OAAO;GAS3C,MAAM,QAAO,MANU,MAAA,QAAM,KAC3B,OAAO,QACP,SACA;IAAE;IAAS,SAAS,OAAO;GAAQ,CACrC,EAAA,CAEsB;GACtB,IAAI,KAAK,YAAY,OACnB,OAAO;IACL,SAAS;IACT,OAAO,0BACL,KAAK,cAAc,OAAO,IAAI,KAAK,WAAW,MAAM,KACnD,KAAK,SAAS;GACnB;GASF,MAAM,YAAY,KAAK,QAAQ,CAAC;GAChC,MAAM,OAA0B,MAAM,QAAQ,SAAS,IACnD,gBAAgB,SAAS,IACxB,UAAU,WAAW;GAuC1B,OAAO;IAAE,SAAS;IAAM,MAAM;KAR5B,UA9ByC,KAAK,OAAO,CAAC,EAAA,CACrD,QAAQ,MAAM,EAAE,OAAO,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAC5C,KAAK,OAAO;MACX,OAAO,EAAE,SAAS;MAClB,MAAM,EAAE;MACR,SAAS,EAAE,eAAe,EAAE,WAAW;MACvC,UAAU,EAAE;KACd,EAuBsB;KACtB,SAtBqC,KAAK,UAAU,CAAC,EAAA,CACpD,QAAQ,MAAM,EAAE,YAAY,QAAQ,EAAE,aAAa,EAAE,CAAC,CACtD,KAAK,OAAO;MACX,OAAO,EAAE;MACT,UAAU,EAAE;MACZ,MAAM,EAAE;MACR,UAAU,EAAE;KACd,EAemB;KACnB,YAAY,CAAC;KACb,QAAQ,CAAC;KACT,OAhBmC,KAAK,QAAQ,CAAC,EAAA,CAChD,QAAQ,MAAM,EAAE,OAAO,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAC5C,KAAK,OAAO;MACX,OAAO,EAAE,SAAS;MAClB,MAAM,EAAE;MACR,SAAS,EAAE,eAAe,EAAE,WAAW;MACvC,MAAM,EAAE;MACR,QAAQ,YAAY,EAAE,GAAa;MACnC,UAAU,EAAE;KACd,EAOgB;KAChB,iBAAiB,CAAC;IAGgB;GAAE;EACxC,SAAS,OAAO;GAGd,OAAO;IACL,SAAS;IACT,OAAO,kCAHP,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK;GAIvD;EACF;CACF;CAEA,OAAO,EAAE,WAAW;AACtB"}
|
|
@@ -2,6 +2,7 @@ const require_runtime = require("../../_virtual/_rolldown/runtime.cjs");
|
|
|
2
2
|
const require_utils = require("./utils.cjs");
|
|
3
3
|
const require_keenable_search = require("./keenable-search.cjs");
|
|
4
4
|
const require_tavily_search = require("./tavily-search.cjs");
|
|
5
|
+
const require_crw_search = require("./crw-search.cjs");
|
|
5
6
|
let axios = require("axios");
|
|
6
7
|
axios = require_runtime.__toESM(axios, 1);
|
|
7
8
|
let _langchain_textsplitters = require("@langchain/textsplitters");
|
|
@@ -274,12 +275,13 @@ const createSearXNGAPI = (instanceUrl, apiKey) => {
|
|
|
274
275
|
return { getSources };
|
|
275
276
|
};
|
|
276
277
|
const createSearchAPI = (config) => {
|
|
277
|
-
const { searchProvider = "serper", serperApiKey, searxngInstanceUrl, searxngApiKey, tavilyApiKey, tavilySearchUrl, tavilySearchOptions, keenableApiKey, keenableApiUrl, keenableSearchOptions } = config;
|
|
278
|
+
const { searchProvider = "serper", serperApiKey, searxngInstanceUrl, searxngApiKey, tavilyApiKey, tavilySearchUrl, tavilySearchOptions, keenableApiKey, keenableApiUrl, keenableSearchOptions, crwApiKey, crwApiUrl, crwSearchOptions } = config;
|
|
278
279
|
if (searchProvider.toLowerCase() === "serper") return createSerperAPI(serperApiKey);
|
|
279
280
|
else if (searchProvider.toLowerCase() === "searxng") return createSearXNGAPI(searxngInstanceUrl, searxngApiKey);
|
|
280
281
|
else if (searchProvider.toLowerCase() === "tavily") return require_tavily_search.createTavilyAPI(tavilyApiKey, tavilySearchUrl, tavilySearchOptions);
|
|
281
282
|
else if (searchProvider.toLowerCase() === "keenable") return require_keenable_search.createKeenableAPI(keenableApiKey, keenableApiUrl, keenableSearchOptions);
|
|
282
|
-
else
|
|
283
|
+
else if (searchProvider.toLowerCase() === "crw") return require_crw_search.createCrwAPI(crwApiKey, crwApiUrl, crwSearchOptions);
|
|
284
|
+
else throw new Error(`Invalid search provider: ${searchProvider}. Must be 'serper', 'searxng', 'tavily', 'keenable', or 'crw'`);
|
|
283
285
|
};
|
|
284
286
|
const createSourceProcessor = (config = {}, scraperInstance) => {
|
|
285
287
|
if (!scraperInstance) throw new Error("Scraper instance is required");
|