@vybestack/llxprt-code-core 0.6.0-nightly.251128.1049d5f2b → 0.6.1-nightly.251129.8e1912779
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/prompt-config/defaults/default-prompts.json +5 -2
- package/dist/src/agents/executor.js +2 -2
- package/dist/src/agents/executor.js.map +1 -1
- package/dist/src/config/config.js +10 -4
- package/dist/src/config/config.js.map +1 -1
- package/dist/src/core/coreToolScheduler.d.ts +7 -0
- package/dist/src/core/coreToolScheduler.js +67 -0
- package/dist/src/core/coreToolScheduler.js.map +1 -1
- package/dist/src/core/prompts.js +9 -4
- package/dist/src/core/prompts.js.map +1 -1
- package/dist/src/index.d.ts +5 -2
- package/dist/src/index.js +5 -2
- package/dist/src/index.js.map +1 -1
- package/dist/src/prompt-config/defaults/tool-defaults.js +5 -2
- package/dist/src/prompt-config/defaults/tool-defaults.js.map +1 -1
- package/dist/src/prompt-config/defaults/tools/code-search.md +3 -0
- package/dist/src/prompt-config/defaults/tools/direct-web-fetch.md +3 -0
- package/dist/src/prompt-config/defaults/tools/exa-web-search.md +14 -0
- package/dist/src/prompt-config/defaults/tools/{web-fetch.md → google-web-fetch.md} +1 -1
- package/dist/src/prompt-config/defaults/tools/{web-search.md → google-web-search.md} +1 -1
- package/dist/src/providers/gemini/GeminiProvider.js +2 -2
- package/dist/src/providers/gemini/GeminiProvider.js.map +1 -1
- package/dist/src/tools/codesearch.d.ts +21 -0
- package/dist/src/tools/codesearch.js +145 -0
- package/dist/src/tools/codesearch.js.map +1 -0
- package/dist/src/tools/direct-web-fetch.d.ts +22 -0
- package/dist/src/tools/direct-web-fetch.js +169 -0
- package/dist/src/tools/direct-web-fetch.js.map +1 -0
- package/dist/src/tools/exa-web-search.d.ts +24 -0
- package/dist/src/tools/exa-web-search.js +137 -0
- package/dist/src/tools/exa-web-search.js.map +1 -0
- package/dist/src/tools/{web-fetch.d.ts → google-web-fetch.d.ts} +4 -4
- package/dist/src/tools/{web-fetch.js → google-web-fetch.js} +29 -16
- package/dist/src/tools/google-web-fetch.js.map +1 -0
- package/dist/src/tools/{web-search-invocation.d.ts → google-web-search-invocation.d.ts} +1 -1
- package/dist/src/tools/{web-search-invocation.js → google-web-search-invocation.js} +2 -2
- package/dist/src/tools/google-web-search-invocation.js.map +1 -0
- package/dist/src/tools/{web-search.d.ts → google-web-search.d.ts} +4 -4
- package/dist/src/tools/{web-search.js → google-web-search.js} +6 -6
- package/dist/src/tools/google-web-search.js.map +1 -0
- package/dist/src/tools/tool-error.d.ts +4 -1
- package/dist/src/tools/tool-error.js +5 -0
- package/dist/src/tools/tool-error.js.map +1 -1
- package/dist/src/utils/fetch.d.ts +1 -1
- package/dist/src/utils/fetch.js +24 -2
- package/dist/src/utils/fetch.js.map +1 -1
- package/package.json +1 -1
- package/dist/src/tools/web-fetch.js.map +0 -1
- package/dist/src/tools/web-search-invocation.js.map +0 -1
- package/dist/src/tools/web-search.js.map +0 -1
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @license
|
|
3
|
+
* Copyright 2025 Vybestack LLC
|
|
4
|
+
* SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
*
|
|
6
|
+
* Portions of this code are derived from opencode (https://github.com/sst/opencode)
|
|
7
|
+
* Copyright (c) 2025 opencode
|
|
8
|
+
* Licensed under the MIT License.
|
|
9
|
+
*/
|
|
10
|
+
import { BaseDeclarativeTool, ToolInvocation, ToolResult } from './tools.js';
|
|
11
|
+
import { Config } from '../config/config.js';
|
|
12
|
+
export interface CodeSearchToolParams {
|
|
13
|
+
query: string;
|
|
14
|
+
tokensNum?: number;
|
|
15
|
+
}
|
|
16
|
+
export declare class CodeSearchTool extends BaseDeclarativeTool<CodeSearchToolParams, ToolResult> {
|
|
17
|
+
private readonly config;
|
|
18
|
+
static readonly Name = "codesearch";
|
|
19
|
+
constructor(config: Config);
|
|
20
|
+
protected createInvocation(params: CodeSearchToolParams): ToolInvocation<CodeSearchToolParams, ToolResult>;
|
|
21
|
+
}
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @license
|
|
3
|
+
* Copyright 2025 Vybestack LLC
|
|
4
|
+
* SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
*
|
|
6
|
+
* Portions of this code are derived from opencode (https://github.com/sst/opencode)
|
|
7
|
+
* Copyright (c) 2025 opencode
|
|
8
|
+
* Licensed under the MIT License.
|
|
9
|
+
*/
|
|
10
|
+
import { BaseDeclarativeTool, Kind, } from './tools.js';
|
|
11
|
+
import { ToolErrorType } from './tool-error.js';
|
|
12
|
+
import { BaseToolInvocation } from './tools.js';
|
|
13
|
+
import fetch from 'node-fetch';
|
|
14
|
+
const API_CONFIG = {
|
|
15
|
+
BASE_URL: 'https://mcp.exa.ai',
|
|
16
|
+
ENDPOINTS: {
|
|
17
|
+
CONTEXT: '/mcp',
|
|
18
|
+
},
|
|
19
|
+
};
|
|
20
|
+
export class CodeSearchTool extends BaseDeclarativeTool {
|
|
21
|
+
config;
|
|
22
|
+
static Name = 'codesearch';
|
|
23
|
+
constructor(config) {
|
|
24
|
+
super(CodeSearchTool.Name, 'CodeSearch', 'Search for relevant code snippets, APIs, Libraries, and SDKs documentation. Use this to find examples and usage patterns.', Kind.Search, {
|
|
25
|
+
type: 'object',
|
|
26
|
+
properties: {
|
|
27
|
+
query: {
|
|
28
|
+
type: 'string',
|
|
29
|
+
description: "Search query to find relevant context. For example, 'React useState hook examples', 'Python pandas dataframe filtering'.",
|
|
30
|
+
},
|
|
31
|
+
tokensNum: {
|
|
32
|
+
type: 'number',
|
|
33
|
+
description: 'Number of tokens to return (1000-50000). Default is 5000 tokens.',
|
|
34
|
+
default: 5000,
|
|
35
|
+
minimum: 1000,
|
|
36
|
+
maximum: 50000,
|
|
37
|
+
},
|
|
38
|
+
},
|
|
39
|
+
required: ['query'],
|
|
40
|
+
});
|
|
41
|
+
this.config = config;
|
|
42
|
+
}
|
|
43
|
+
createInvocation(params) {
|
|
44
|
+
return new CodeSearchToolInvocation(this.config, params);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
class CodeSearchToolInvocation extends BaseToolInvocation {
|
|
48
|
+
config;
|
|
49
|
+
constructor(config, params) {
|
|
50
|
+
super(params);
|
|
51
|
+
this.config = config;
|
|
52
|
+
}
|
|
53
|
+
getDescription() {
|
|
54
|
+
return `Search code for: ${this.params.query}`;
|
|
55
|
+
}
|
|
56
|
+
async execute(signal, _updateOutput) {
|
|
57
|
+
const codeRequest = {
|
|
58
|
+
jsonrpc: '2.0',
|
|
59
|
+
id: 1,
|
|
60
|
+
method: 'tools/call',
|
|
61
|
+
params: {
|
|
62
|
+
name: 'get_code_context_exa',
|
|
63
|
+
arguments: {
|
|
64
|
+
query: this.params.query,
|
|
65
|
+
tokensNum: this.getEffectiveTokensNum(),
|
|
66
|
+
},
|
|
67
|
+
},
|
|
68
|
+
};
|
|
69
|
+
try {
|
|
70
|
+
const headers = {
|
|
71
|
+
accept: 'application/json, text/event-stream',
|
|
72
|
+
'content-type': 'application/json',
|
|
73
|
+
};
|
|
74
|
+
const response = await fetch(`${API_CONFIG.BASE_URL}${API_CONFIG.ENDPOINTS.CONTEXT}`, {
|
|
75
|
+
method: 'POST',
|
|
76
|
+
headers,
|
|
77
|
+
body: JSON.stringify(codeRequest),
|
|
78
|
+
signal,
|
|
79
|
+
});
|
|
80
|
+
if (!response.ok) {
|
|
81
|
+
const errorText = await response.text();
|
|
82
|
+
throw new Error(`Code search error (${response.status}): ${errorText}`);
|
|
83
|
+
}
|
|
84
|
+
const responseText = await response.text();
|
|
85
|
+
// Parse SSE response
|
|
86
|
+
// The response from mcp.exa.ai seems to be SSE format "data: {...}"
|
|
87
|
+
const lines = responseText.split('\n');
|
|
88
|
+
for (const line of lines) {
|
|
89
|
+
if (line.startsWith('data: ')) {
|
|
90
|
+
try {
|
|
91
|
+
const data = JSON.parse(line.substring(6));
|
|
92
|
+
if (data.result &&
|
|
93
|
+
data.result.content &&
|
|
94
|
+
data.result.content.length > 0) {
|
|
95
|
+
const content = data.result.content[0].text;
|
|
96
|
+
return {
|
|
97
|
+
llmContent: content,
|
|
98
|
+
returnDisplay: content,
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
catch (_e) {
|
|
103
|
+
// Ignore parse errors for intermediate lines
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
return {
|
|
108
|
+
llmContent: 'No code snippets or documentation found. Please try a different query.',
|
|
109
|
+
returnDisplay: 'No results found.',
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
catch (error) {
|
|
113
|
+
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
114
|
+
return {
|
|
115
|
+
llmContent: `Error performing code search: ${errorMessage}`,
|
|
116
|
+
returnDisplay: `Error: ${errorMessage}`,
|
|
117
|
+
error: {
|
|
118
|
+
message: errorMessage,
|
|
119
|
+
type: ToolErrorType.SEARCH_ERROR,
|
|
120
|
+
},
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
getEffectiveTokensNum() {
|
|
125
|
+
// Limits from: https://github.com/exa-labs/exa-mcp-server/blob/1ec2078b59d5e4503696fecf9ce40701ccef85cc/src/tools/exaCode.ts#L14
|
|
126
|
+
const LIMITS = {
|
|
127
|
+
DEFAULT: 5000,
|
|
128
|
+
MIN: 1000,
|
|
129
|
+
MAX: 50000,
|
|
130
|
+
};
|
|
131
|
+
const settingMaxTokens = this.config
|
|
132
|
+
.getSettingsService()
|
|
133
|
+
.get('tool-output-max-tokens');
|
|
134
|
+
// 1. Determine requested tokens (Param > Default)
|
|
135
|
+
let tokens = this.params.tokensNum ?? LIMITS.DEFAULT;
|
|
136
|
+
// 2. Apply setting as a hard cap if present
|
|
137
|
+
if (settingMaxTokens !== undefined &&
|
|
138
|
+
typeof settingMaxTokens === 'number') {
|
|
139
|
+
tokens = Math.min(tokens, settingMaxTokens);
|
|
140
|
+
}
|
|
141
|
+
// 3. Clamp to absolute API limits
|
|
142
|
+
return Math.max(LIMITS.MIN, Math.min(tokens, LIMITS.MAX));
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
//# sourceMappingURL=codesearch.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"codesearch.js","sourceRoot":"","sources":["../../../src/tools/codesearch.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EACL,mBAAmB,EACnB,IAAI,GAGL,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAEhD,OAAO,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,KAAK,MAAM,YAAY,CAAC;AAE/B,MAAM,UAAU,GAAG;IACjB,QAAQ,EAAE,oBAAoB;IAC9B,SAAS,EAAE;QACT,OAAO,EAAE,MAAM;KAChB;CACO,CAAC;AA8BX,MAAM,OAAO,cAAe,SAAQ,mBAGnC;IAG8B;IAF7B,MAAM,CAAU,IAAI,GAAG,YAAY,CAAC;IAEpC,YAA6B,MAAc;QACzC,KAAK,CACH,cAAc,CAAC,IAAI,EACnB,YAAY,EACZ,2HAA2H,EAC3H,IAAI,CAAC,MAAM,EACX;YACE,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,KAAK,EAAE;oBACL,IAAI,EAAE,QAAQ;oBACd,WAAW,EACT,0HAA0H;iBAC7H;gBACD,SAAS,EAAE;oBACT,IAAI,EAAE,QAAQ;oBACd,WAAW,EACT,kEAAkE;oBACpE,OAAO,EAAE,IAAI;oBACb,OAAO,EAAE,IAAI;oBACb,OAAO,EAAE,KAAK;iBACf;aACF;YACD,QAAQ,EAAE,CAAC,OAAO,CAAC;SACpB,CACF,CAAC;QAzByB,WAAM,GAAN,MAAM,CAAQ;IA0B3C,CAAC;IAES,gBAAgB,CACxB,MAA4B;QAE5B,OAAO,IAAI,wBAAwB,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC3D,CAAC;;AAGH,MAAM,wBAAyB,SAAQ,kBAGtC;IAEoB;IADnB,YACmB,MAAc,EAC/B,MAA4B;QAE5B,KAAK,CAAC,MAAM,CAAC,CAAC;QAHG,WAAM,GAAN,MAAM,CAAQ;IAIjC,CAAC;IAED,cAAc;QACZ,OAAO,oBAAoB,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IACjD,CAAC;IAED,KAAK,CAAC,OAAO,CACX,MAAmB,EACnB,aAAwC;QAExC,MAAM,WAAW,GAAmB;YAClC,OAAO,EAAE,KAAK;YACd,EAAE,EAAE,CAAC;YACL,MAAM,EAAE,YAAY;YACpB,MAAM,EAAE;gBACN,IAAI,EAAE,sBAAsB;gBAC5B,SAAS,EAAE;oBACT,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK;oBACxB,SAAS,EAAE,IAAI,CAAC,qBAAqB,EAAE;iBACxC;aACF;SACF,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,OAAO,GAA2B;gBACtC,MAAM,EAAE,qCAAqC;gBAC7C,cAAc,EAAE,kBAAkB;aACnC,CAAC;YAEF,MAAM,QAAQ,GAAG,MAAM,KAAK,CAC1B,GAAG,UAAU,CAAC,QAAQ,GAAG,UAAU,CAAC,SAAS,CAAC,OAAO,EAAE,EACvD;gBACE,MAAM,EAAE,MAAM;gBACd,OAAO;gBACP,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC;gBACjC,MAAM;aACP,CACF,CAAC;YAEF,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACxC,MAAM,IAAI,KAAK,CAAC,sBAAsB,QAAQ,CAAC,MAAM,MAAM,SAAS,EAAE,CAAC,CAAC;YAC1E,CAAC;YAED,MAAM,YAAY,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YAE3C,qBAAqB;YACrB,oEAAoE;YACpE,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACvC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,IAAI,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAC9B,IAAI,CAAC;wBACH,MAAM,IAAI,GAAoB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;wBAC5D,IACE,IAAI,CAAC,MAAM;4BACX,IAAI,CAAC,MAAM,CAAC,OAAO;4BACnB,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAC9B,CAAC;4BACD,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;4BAC5C,OAAO;gCACL,UAAU,EAAE,OAAO;gCACnB,aAAa,EAAE,OAAO;6BACvB,CAAC;wBACJ,CAAC;oBACH,CAAC;oBAAC,OAAO,EAAE,EAAE,CAAC;wBACZ,6CAA6C;oBAC/C,CAAC;gBACH,CAAC;YACH,CAAC;YAED,OAAO;gBACL,UAAU,EACR,wEAAwE;gBAC1E,aAAa,EAAE,mBAAmB;aACnC,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACzD,OAAO;gBACL,UAAU,EAAE,iCAAiC,YAAY,EAAE;gBAC3D,aAAa,EAAE,UAAU,YAAY,EAAE;gBACvC,KAAK,EAAE;oBACL,OAAO,EAAE,YAAY;oBACrB,IAAI,EAAE,aAAa,CAAC,YAAY;iBACjC;aACF,CAAC;QACJ,CAAC;IACH,CAAC;IAEO,qBAAqB;QAC3B,iIAAiI;QACjI,MAAM,MAAM,GAAG;YACb,OAAO,EAAE,IAAI;YACb,GAAG,EAAE,IAAI;YACT,GAAG,EAAE,KAAK;SACX,CAAC;QAEF,MAAM,gBAAgB,GAAG,IAAI,CAAC,MAAM;aACjC,kBAAkB,EAAE;aACpB,GAAG,CAAC,wBAAwB,CAAuB,CAAC;QAEvD,kDAAkD;QAClD,IAAI,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,MAAM,CAAC,OAAO,CAAC;QAErD,4CAA4C;QAC5C,IACE,gBAAgB,KAAK,SAAS;YAC9B,OAAO,gBAAgB,KAAK,QAAQ,EACpC,CAAC;YACD,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;QAC9C,CAAC;QAED,kCAAkC;QAClC,OAAO,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;IAC5D,CAAC;CACF"}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @license
|
|
3
|
+
* Copyright 2025 Vybestack LLC
|
|
4
|
+
* SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
*
|
|
6
|
+
* Portions of this code are derived from opencode (https://github.com/sst/opencode)
|
|
7
|
+
* Copyright (c) 2025 opencode
|
|
8
|
+
* Licensed under the MIT License.
|
|
9
|
+
*/
|
|
10
|
+
import { BaseDeclarativeTool, ToolInvocation, ToolResult } from './tools.js';
|
|
11
|
+
import { Config } from '../config/config.js';
|
|
12
|
+
export interface DirectWebFetchToolParams {
|
|
13
|
+
url: string;
|
|
14
|
+
format: 'text' | 'markdown' | 'html';
|
|
15
|
+
timeout?: number;
|
|
16
|
+
}
|
|
17
|
+
export declare class DirectWebFetchTool extends BaseDeclarativeTool<DirectWebFetchToolParams, ToolResult> {
|
|
18
|
+
private readonly config;
|
|
19
|
+
static readonly Name = "direct_web_fetch";
|
|
20
|
+
constructor(config: Config);
|
|
21
|
+
protected createInvocation(params: DirectWebFetchToolParams): ToolInvocation<DirectWebFetchToolParams, ToolResult>;
|
|
22
|
+
}
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @license
|
|
3
|
+
* Copyright 2025 Vybestack LLC
|
|
4
|
+
* SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
*
|
|
6
|
+
* Portions of this code are derived from opencode (https://github.com/sst/opencode)
|
|
7
|
+
* Copyright (c) 2025 opencode
|
|
8
|
+
* Licensed under the MIT License.
|
|
9
|
+
*/
|
|
10
|
+
import { BaseDeclarativeTool, Kind, } from './tools.js';
|
|
11
|
+
import { ToolErrorType } from './tool-error.js';
|
|
12
|
+
import { BaseToolInvocation } from './tools.js';
|
|
13
|
+
import fetch from 'node-fetch';
|
|
14
|
+
import TurndownService from 'turndown';
|
|
15
|
+
import * as cheerio from 'cheerio';
|
|
16
|
+
const MAX_RESPONSE_SIZE = 5 * 1024 * 1024; // 5MB
|
|
17
|
+
const DEFAULT_TIMEOUT = 30 * 1000; // 30 seconds
|
|
18
|
+
const MAX_TIMEOUT = 120 * 1000; // 2 minutes
|
|
19
|
+
export class DirectWebFetchTool extends BaseDeclarativeTool {
|
|
20
|
+
config;
|
|
21
|
+
static Name = 'direct_web_fetch';
|
|
22
|
+
constructor(config) {
|
|
23
|
+
super(DirectWebFetchTool.Name, 'DirectWebFetch', 'Fetches content from a specified URL and converts it to the requested format (text, markdown, or html).', Kind.Search, {
|
|
24
|
+
type: 'object',
|
|
25
|
+
properties: {
|
|
26
|
+
url: {
|
|
27
|
+
type: 'string',
|
|
28
|
+
description: 'The URL to fetch content from',
|
|
29
|
+
},
|
|
30
|
+
format: {
|
|
31
|
+
type: 'string',
|
|
32
|
+
enum: ['text', 'markdown', 'html'],
|
|
33
|
+
description: 'The format to return the content in (text, markdown, or html)',
|
|
34
|
+
},
|
|
35
|
+
timeout: {
|
|
36
|
+
type: 'number',
|
|
37
|
+
description: 'Optional timeout in seconds (max 120)',
|
|
38
|
+
},
|
|
39
|
+
},
|
|
40
|
+
required: ['url', 'format'],
|
|
41
|
+
});
|
|
42
|
+
this.config = config;
|
|
43
|
+
}
|
|
44
|
+
createInvocation(params) {
|
|
45
|
+
return new DirectWebFetchToolInvocation(this.config, params);
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
class DirectWebFetchToolInvocation extends BaseToolInvocation {
|
|
49
|
+
constructor(_config, params, messageBus) {
|
|
50
|
+
super(params, messageBus);
|
|
51
|
+
}
|
|
52
|
+
getDescription() {
|
|
53
|
+
return `Fetch content from ${this.params.url}`;
|
|
54
|
+
}
|
|
55
|
+
async execute(signal, _updateOutput) {
|
|
56
|
+
const { url, format, timeout: timeoutSec } = this.params;
|
|
57
|
+
if (!url.startsWith('http://') && !url.startsWith('https://')) {
|
|
58
|
+
return {
|
|
59
|
+
llmContent: 'URL must start with http:// or https://',
|
|
60
|
+
returnDisplay: 'Invalid URL',
|
|
61
|
+
error: {
|
|
62
|
+
message: 'Invalid URL protocol',
|
|
63
|
+
type: ToolErrorType.INVALID_ARGUMENT,
|
|
64
|
+
},
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
const timeout = Math.min((timeoutSec ?? DEFAULT_TIMEOUT / 1000) * 1000, MAX_TIMEOUT);
|
|
68
|
+
// Build Accept header
|
|
69
|
+
let acceptHeader = '*/*';
|
|
70
|
+
switch (format) {
|
|
71
|
+
case 'markdown':
|
|
72
|
+
acceptHeader =
|
|
73
|
+
'text/markdown;q=1.0, text/x-markdown;q=0.9, text/plain;q=0.8, text/html;q=0.7, */*;q=0.1';
|
|
74
|
+
break;
|
|
75
|
+
case 'text':
|
|
76
|
+
acceptHeader =
|
|
77
|
+
'text/plain;q=1.0, text/markdown;q=0.9, text/html;q=0.8, */*;q=0.1';
|
|
78
|
+
break;
|
|
79
|
+
case 'html':
|
|
80
|
+
acceptHeader =
|
|
81
|
+
'text/html;q=1.0, application/xhtml+xml;q=0.9, text/plain;q=0.8, text/markdown;q=0.7, */*;q=0.1';
|
|
82
|
+
break;
|
|
83
|
+
default:
|
|
84
|
+
acceptHeader =
|
|
85
|
+
'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8';
|
|
86
|
+
}
|
|
87
|
+
const controller = new AbortController();
|
|
88
|
+
const timeoutId = setTimeout(() => controller.abort(), timeout);
|
|
89
|
+
// If the parent signal aborts, we should also abort our controller
|
|
90
|
+
const onAbort = () => controller.abort();
|
|
91
|
+
signal.addEventListener('abort', onAbort);
|
|
92
|
+
try {
|
|
93
|
+
const response = await fetch(url, {
|
|
94
|
+
signal: controller.signal,
|
|
95
|
+
headers: {
|
|
96
|
+
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
|
97
|
+
Accept: acceptHeader,
|
|
98
|
+
'Accept-Language': 'en-US,en;q=0.9',
|
|
99
|
+
},
|
|
100
|
+
});
|
|
101
|
+
if (!response.ok) {
|
|
102
|
+
throw new Error(`Request failed with status code: ${response.status}`);
|
|
103
|
+
}
|
|
104
|
+
// Check content length
|
|
105
|
+
const contentLength = response.headers.get('content-length');
|
|
106
|
+
if (contentLength && parseInt(contentLength, 10) > MAX_RESPONSE_SIZE) {
|
|
107
|
+
throw new Error('Response too large (exceeds 5MB limit)');
|
|
108
|
+
}
|
|
109
|
+
const arrayBuffer = await response.arrayBuffer();
|
|
110
|
+
if (arrayBuffer.byteLength > MAX_RESPONSE_SIZE) {
|
|
111
|
+
throw new Error('Response too large (exceeds 5MB limit)');
|
|
112
|
+
}
|
|
113
|
+
const content = new TextDecoder().decode(arrayBuffer);
|
|
114
|
+
const contentType = response.headers.get('content-type') || '';
|
|
115
|
+
let output = content;
|
|
116
|
+
switch (format) {
|
|
117
|
+
case 'markdown':
|
|
118
|
+
if (contentType.includes('text/html')) {
|
|
119
|
+
output = this.convertHTMLToMarkdown(content);
|
|
120
|
+
}
|
|
121
|
+
break;
|
|
122
|
+
case 'text':
|
|
123
|
+
if (contentType.includes('text/html')) {
|
|
124
|
+
output = this.extractTextFromHTML(content);
|
|
125
|
+
}
|
|
126
|
+
break;
|
|
127
|
+
default:
|
|
128
|
+
break;
|
|
129
|
+
}
|
|
130
|
+
return {
|
|
131
|
+
llmContent: output,
|
|
132
|
+
returnDisplay: `Fetched ${url} as ${format}`,
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
catch (error) {
|
|
136
|
+
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
137
|
+
return {
|
|
138
|
+
llmContent: `Error fetching URL: ${errorMessage}`,
|
|
139
|
+
returnDisplay: `Error: ${errorMessage}`,
|
|
140
|
+
error: {
|
|
141
|
+
message: errorMessage,
|
|
142
|
+
type: ToolErrorType.FETCH_ERROR,
|
|
143
|
+
},
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
finally {
|
|
147
|
+
clearTimeout(timeoutId);
|
|
148
|
+
signal.removeEventListener('abort', onAbort);
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
extractTextFromHTML(html) {
|
|
152
|
+
const $ = cheerio.load(html);
|
|
153
|
+
// Remove scripts, styles, etc.
|
|
154
|
+
$('script, style, noscript, iframe, object, embed').remove();
|
|
155
|
+
return $.text().trim();
|
|
156
|
+
}
|
|
157
|
+
convertHTMLToMarkdown(html) {
|
|
158
|
+
const turndownService = new TurndownService({
|
|
159
|
+
headingStyle: 'atx',
|
|
160
|
+
hr: '---',
|
|
161
|
+
bulletListMarker: '-',
|
|
162
|
+
codeBlockStyle: 'fenced',
|
|
163
|
+
emDelimiter: '*',
|
|
164
|
+
});
|
|
165
|
+
turndownService.remove(['script', 'style', 'meta', 'link']);
|
|
166
|
+
return turndownService.turndown(html);
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
//# sourceMappingURL=direct-web-fetch.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"direct-web-fetch.js","sourceRoot":"","sources":["../../../src/tools/direct-web-fetch.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAGH,OAAO,EACL,mBAAmB,EACnB,IAAI,GAGL,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAEhD,OAAO,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,KAAsB,MAAM,YAAY,CAAC;AAChD,OAAO,eAAe,MAAM,UAAU,CAAC;AACvC,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,MAAM,iBAAiB,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,MAAM;AACjD,MAAM,eAAe,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,aAAa;AAChD,MAAM,WAAW,GAAG,GAAG,GAAG,IAAI,CAAC,CAAC,YAAY;AAQ5C,MAAM,OAAO,kBAAmB,SAAQ,mBAGvC;IAG8B;IAF7B,MAAM,CAAU,IAAI,GAAG,kBAAkB,CAAC;IAE1C,YAA6B,MAAc;QACzC,KAAK,CACH,kBAAkB,CAAC,IAAI,EACvB,gBAAgB,EAChB,yGAAyG,EACzG,IAAI,CAAC,MAAM,EACX;YACE,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,GAAG,EAAE;oBACH,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,+BAA+B;iBAC7C;gBACD,MAAM,EAAE;oBACN,IAAI,EAAE,QAAQ;oBACd,IAAI,EAAE,CAAC,MAAM,EAAE,UAAU,EAAE,MAAM,CAAC;oBAClC,WAAW,EACT,+DAA+D;iBAClE;gBACD,OAAO,EAAE;oBACP,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,uCAAuC;iBACrD;aACF;YACD,QAAQ,EAAE,CAAC,KAAK,EAAE,QAAQ,CAAC;SAC5B,CACF,CAAC;QA1ByB,WAAM,GAAN,MAAM,CAAQ;IA2B3C,CAAC;IAES,gBAAgB,CACxB,MAAgC;QAEhC,OAAO,IAAI,4BAA4B,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC/D,CAAC;;AAGH,MAAM,4BAA6B,SAAQ,kBAG1C;IACC,YACE,OAAe,EACf,MAAgC,EAChC,UAAuB;QAEvB,KAAK,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IAC5B,CAAC;IAED,cAAc;QACZ,OAAO,sBAAsB,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;IACjD,CAAC;IAED,KAAK,CAAC,OAAO,CACX,MAAmB,EACnB,aAAwC;QAExC,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC;QAEzD,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAC9D,OAAO;gBACL,UAAU,EAAE,yCAAyC;gBACrD,aAAa,EAAE,aAAa;gBAC5B,KAAK,EAAE;oBACL,OAAO,EAAE,sBAAsB;oBAC/B,IAAI,EAAE,aAAa,CAAC,gBAAgB;iBACrC;aACF,CAAC;QACJ,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CACtB,CAAC,UAAU,IAAI,eAAe,GAAG,IAAI,CAAC,GAAG,IAAI,EAC7C,WAAW,CACZ,CAAC;QAEF,sBAAsB;QACtB,IAAI,YAAY,GAAG,KAAK,CAAC;QACzB,QAAQ,MAAM,EAAE,CAAC;YACf,KAAK,UAAU;gBACb,YAAY;oBACV,0FAA0F,CAAC;gBAC7F,MAAM;YACR,KAAK,MAAM;gBACT,YAAY;oBACV,mEAAmE,CAAC;gBACtE,MAAM;YACR,KAAK,MAAM;gBACT,YAAY;oBACV,gGAAgG,CAAC;gBACnG,MAAM;YACR;gBACE,YAAY;oBACV,kGAAkG,CAAC;QACzG,CAAC;QAED,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,OAAO,CAAC,CAAC;QAEhE,mEAAmE;QACnE,MAAM,OAAO,GAAG,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;QACzC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAE1C,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;gBAChC,MAAM,EAAE,UAAU,CAAC,MAAM;gBACzB,OAAO,EAAE;oBACP,YAAY,EACV,iHAAiH;oBACnH,MAAM,EAAE,YAAY;oBACpB,iBAAiB,EAAE,gBAAgB;iBACpC;aACa,CAAC,CAAC;YAElB,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,IAAI,KAAK,CAAC,oCAAoC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;YACzE,CAAC;YAED,uBAAuB;YACvB,MAAM,aAAa,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;YAC7D,IAAI,aAAa,IAAI,QAAQ,CAAC,aAAa,EAAE,EAAE,CAAC,GAAG,iBAAiB,EAAE,CAAC;gBACrE,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;YAC5D,CAAC;YAED,MAAM,WAAW,GAAG,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC;YACjD,IAAI,WAAW,CAAC,UAAU,GAAG,iBAAiB,EAAE,CAAC;gBAC/C,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;YAC5D,CAAC;YAED,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;YACtD,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YAE/D,IAAI,MAAM,GAAG,OAAO,CAAC;YAErB,QAAQ,MAAM,EAAE,CAAC;gBACf,KAAK,UAAU;oBACb,IAAI,WAAW,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;wBACtC,MAAM,GAAG,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC;oBAC/C,CAAC;oBACD,MAAM;gBAER,KAAK,MAAM;oBACT,IAAI,WAAW,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;wBACtC,MAAM,GAAG,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;oBAC7C,CAAC;oBACD,MAAM;gBACR;oBACE,MAAM;YACV,CAAC;YAED,OAAO;gBACL,UAAU,EAAE,MAAM;gBAClB,aAAa,EAAE,WAAW,GAAG,OAAO,MAAM,EAAE;aAC7C,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACzD,OAAO;gBACL,UAAU,EAAE,uBAAuB,YAAY,EAAE;gBACjD,aAAa,EAAE,UAAU,YAAY,EAAE;gBACvC,KAAK,EAAE;oBACL,OAAO,EAAE,YAAY;oBACrB,IAAI,EAAE,aAAa,CAAC,WAAW;iBAChC;aACF,CAAC;QACJ,CAAC;gBAAS,CAAC;YACT,YAAY,CAAC,SAAS,CAAC,CAAC;YACxB,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;IAEO,mBAAmB,CAAC,IAAY;QACtC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,+BAA+B;QAC/B,CAAC,CAAC,gDAAgD,CAAC,CAAC,MAAM,EAAE,CAAC;QAC7D,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;IACzB,CAAC;IAEO,qBAAqB,CAAC,IAAY;QACxC,MAAM,eAAe,GAAG,IAAI,eAAe,CAAC;YAC1C,YAAY,EAAE,KAAK;YACnB,EAAE,EAAE,KAAK;YACT,gBAAgB,EAAE,GAAG;YACrB,cAAc,EAAE,QAAQ;YACxB,WAAW,EAAE,GAAG;SACjB,CAAC,CAAC;QACH,eAAe,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;QAC5D,OAAO,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IACxC,CAAC;CACF"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @license
|
|
3
|
+
* Copyright 2025 Vybestack LLC
|
|
4
|
+
* SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
*
|
|
6
|
+
* Portions of this code are derived from opencode (https://github.com/sst/opencode)
|
|
7
|
+
* Copyright (c) 2025 opencode
|
|
8
|
+
* Licensed under the MIT License.
|
|
9
|
+
*/
|
|
10
|
+
import { BaseDeclarativeTool, ToolInvocation, ToolResult } from './tools.js';
|
|
11
|
+
import { Config } from '../config/config.js';
|
|
12
|
+
export interface ExaWebSearchToolParams {
|
|
13
|
+
query: string;
|
|
14
|
+
numResults?: number;
|
|
15
|
+
livecrawl?: 'fallback' | 'preferred';
|
|
16
|
+
type?: 'auto' | 'fast' | 'deep';
|
|
17
|
+
contextMaxCharacters?: number;
|
|
18
|
+
}
|
|
19
|
+
export declare class ExaWebSearchTool extends BaseDeclarativeTool<ExaWebSearchToolParams, ToolResult> {
|
|
20
|
+
private readonly config;
|
|
21
|
+
static readonly Name = "exa_web_search";
|
|
22
|
+
constructor(config: Config);
|
|
23
|
+
protected createInvocation(params: ExaWebSearchToolParams): ToolInvocation<ExaWebSearchToolParams, ToolResult>;
|
|
24
|
+
}
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @license
|
|
3
|
+
* Copyright 2025 Vybestack LLC
|
|
4
|
+
* SPDX-License-Identifier: Apache-2.0
|
|
5
|
+
*
|
|
6
|
+
* Portions of this code are derived from opencode (https://github.com/sst/opencode)
|
|
7
|
+
* Copyright (c) 2025 opencode
|
|
8
|
+
* Licensed under the MIT License.
|
|
9
|
+
*/
|
|
10
|
+
import { BaseDeclarativeTool, Kind, } from './tools.js';
|
|
11
|
+
import { ToolErrorType } from './tool-error.js';
|
|
12
|
+
import { BaseToolInvocation } from './tools.js';
|
|
13
|
+
import fetch from 'node-fetch';
|
|
14
|
+
const API_CONFIG = {
|
|
15
|
+
BASE_URL: 'https://mcp.exa.ai',
|
|
16
|
+
ENDPOINTS: {
|
|
17
|
+
SEARCH: '/mcp',
|
|
18
|
+
},
|
|
19
|
+
DEFAULT_NUM_RESULTS: 8,
|
|
20
|
+
};
|
|
21
|
+
export class ExaWebSearchTool extends BaseDeclarativeTool {
|
|
22
|
+
config;
|
|
23
|
+
static Name = 'exa_web_search';
|
|
24
|
+
constructor(config) {
|
|
25
|
+
super(ExaWebSearchTool.Name, 'ExaWebSearch', 'Search the web using Exa AI - performs real-time web searches and can scrape content from specific URLs. Provides up-to-date information for current events and recent data.', Kind.Search, {
|
|
26
|
+
type: 'object',
|
|
27
|
+
properties: {
|
|
28
|
+
query: {
|
|
29
|
+
type: 'string',
|
|
30
|
+
description: 'Websearch query',
|
|
31
|
+
},
|
|
32
|
+
numResults: {
|
|
33
|
+
type: 'number',
|
|
34
|
+
description: 'Number of search results to return (default: 8)',
|
|
35
|
+
},
|
|
36
|
+
livecrawl: {
|
|
37
|
+
type: 'string',
|
|
38
|
+
enum: ['fallback', 'preferred'],
|
|
39
|
+
description: "Live crawl mode - 'fallback': use live crawling as backup if cached content unavailable, 'preferred': prioritize live crawling (default: 'fallback')",
|
|
40
|
+
},
|
|
41
|
+
type: {
|
|
42
|
+
type: 'string',
|
|
43
|
+
enum: ['auto', 'fast', 'deep'],
|
|
44
|
+
description: "Search type - 'auto': balanced search (default), 'fast': quick results, 'deep': comprehensive search",
|
|
45
|
+
},
|
|
46
|
+
contextMaxCharacters: {
|
|
47
|
+
type: 'number',
|
|
48
|
+
description: 'Maximum characters for context string optimized for LLMs (default: 10000)',
|
|
49
|
+
},
|
|
50
|
+
},
|
|
51
|
+
required: ['query'],
|
|
52
|
+
});
|
|
53
|
+
this.config = config;
|
|
54
|
+
}
|
|
55
|
+
createInvocation(params) {
|
|
56
|
+
return new ExaWebSearchToolInvocation(this.config, params);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
class ExaWebSearchToolInvocation extends BaseToolInvocation {
|
|
60
|
+
constructor(_config, params) {
|
|
61
|
+
super(params);
|
|
62
|
+
}
|
|
63
|
+
getDescription() {
|
|
64
|
+
return `Search web for: ${this.params.query}`;
|
|
65
|
+
}
|
|
66
|
+
async execute(signal, _updateOutput) {
|
|
67
|
+
const searchRequest = {
|
|
68
|
+
jsonrpc: '2.0',
|
|
69
|
+
id: 1,
|
|
70
|
+
method: 'tools/call',
|
|
71
|
+
params: {
|
|
72
|
+
name: 'web_search_exa',
|
|
73
|
+
arguments: {
|
|
74
|
+
query: this.params.query,
|
|
75
|
+
type: this.params.type || 'auto',
|
|
76
|
+
numResults: this.params.numResults || API_CONFIG.DEFAULT_NUM_RESULTS,
|
|
77
|
+
livecrawl: this.params.livecrawl || 'fallback',
|
|
78
|
+
contextMaxCharacters: this.params.contextMaxCharacters ?? 10000,
|
|
79
|
+
},
|
|
80
|
+
},
|
|
81
|
+
};
|
|
82
|
+
try {
|
|
83
|
+
const headers = {
|
|
84
|
+
accept: 'application/json, text/event-stream',
|
|
85
|
+
'content-type': 'application/json',
|
|
86
|
+
};
|
|
87
|
+
const response = await fetch(`${API_CONFIG.BASE_URL}${API_CONFIG.ENDPOINTS.SEARCH}`, {
|
|
88
|
+
method: 'POST',
|
|
89
|
+
headers,
|
|
90
|
+
body: JSON.stringify(searchRequest),
|
|
91
|
+
signal,
|
|
92
|
+
});
|
|
93
|
+
if (!response.ok) {
|
|
94
|
+
const errorText = await response.text();
|
|
95
|
+
throw new Error(`Search error (${response.status}): ${errorText}`);
|
|
96
|
+
}
|
|
97
|
+
const responseText = await response.text();
|
|
98
|
+
// Parse SSE response
|
|
99
|
+
const lines = responseText.split('\n');
|
|
100
|
+
for (const line of lines) {
|
|
101
|
+
if (line.startsWith('data: ')) {
|
|
102
|
+
try {
|
|
103
|
+
const data = JSON.parse(line.substring(6));
|
|
104
|
+
if (data.result &&
|
|
105
|
+
data.result.content &&
|
|
106
|
+
data.result.content.length > 0) {
|
|
107
|
+
const content = data.result.content[0].text;
|
|
108
|
+
return {
|
|
109
|
+
llmContent: content,
|
|
110
|
+
returnDisplay: content,
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
catch (_e) {
|
|
115
|
+
// Ignore parse errors
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
return {
|
|
120
|
+
llmContent: 'No search results found. Please try a different query.',
|
|
121
|
+
returnDisplay: 'No results found.',
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
catch (error) {
|
|
125
|
+
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
126
|
+
return {
|
|
127
|
+
llmContent: `Error performing web search: ${errorMessage}`,
|
|
128
|
+
returnDisplay: `Error: ${errorMessage}`,
|
|
129
|
+
error: {
|
|
130
|
+
message: errorMessage,
|
|
131
|
+
type: ToolErrorType.WEB_SEARCH_FAILED,
|
|
132
|
+
},
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
//# sourceMappingURL=exa-web-search.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"exa-web-search.js","sourceRoot":"","sources":["../../../src/tools/exa-web-search.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EACL,mBAAmB,EACnB,IAAI,GAGL,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAEhD,OAAO,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,KAAK,MAAM,YAAY,CAAC;AAE/B,MAAM,UAAU,GAAG;IACjB,QAAQ,EAAE,oBAAoB;IAC9B,SAAS,EAAE;QACT,MAAM,EAAE,MAAM;KACf;IACD,mBAAmB,EAAE,CAAC;CACd,CAAC;AAoCX,MAAM,OAAO,gBAAiB,SAAQ,mBAGrC;IAG8B;IAF7B,MAAM,CAAU,IAAI,GAAG,gBAAgB,CAAC;IAExC,YAA6B,MAAc;QACzC,KAAK,CACH,gBAAgB,CAAC,IAAI,EACrB,cAAc,EACd,8KAA8K,EAC9K,IAAI,CAAC,MAAM,EACX;YACE,IAAI,EAAE,QAAQ;YACd,UAAU,EAAE;gBACV,KAAK,EAAE;oBACL,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,iBAAiB;iBAC/B;gBACD,UAAU,EAAE;oBACV,IAAI,EAAE,QAAQ;oBACd,WAAW,EAAE,iDAAiD;iBAC/D;gBACD,SAAS,EAAE;oBACT,IAAI,EAAE,QAAQ;oBACd,IAAI,EAAE,CAAC,UAAU,EAAE,WAAW,CAAC;oBAC/B,WAAW,EACT,sJAAsJ;iBACzJ;gBACD,IAAI,EAAE;oBACJ,IAAI,EAAE,QAAQ;oBACd,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;oBAC9B,WAAW,EACT,sGAAsG;iBACzG;gBACD,oBAAoB,EAAE;oBACpB,IAAI,EAAE,QAAQ;oBACd,WAAW,EACT,2EAA2E;iBAC9E;aACF;YACD,QAAQ,EAAE,CAAC,OAAO,CAAC;SACpB,CACF,CAAC;QArCyB,WAAM,GAAN,MAAM,CAAQ;IAsC3C,CAAC;IAES,gBAAgB,CACxB,MAA8B;QAE9B,OAAO,IAAI,0BAA0B,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7D,CAAC;;AAGH,MAAM,0BAA2B,SAAQ,kBAGxC;IACC,YAAY,OAAe,EAAE,MAA8B;QACzD,KAAK,CAAC,MAAM,CAAC,CAAC;IAChB,CAAC;IAED,cAAc;QACZ,OAAO,mBAAmB,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,OAAO,CACX,MAAmB,EACnB,aAAwC;QAExC,MAAM,aAAa,GAAqB;YACtC,OAAO,EAAE,KAAK;YACd,EAAE,EAAE,CAAC;YACL,MAAM,EAAE,YAAY;YACpB,MAAM,EAAE;gBACN,IAAI,EAAE,gBAAgB;gBACtB,SAAS,EAAE;oBACT,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK;oBACxB,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,IAAI,MAAM;oBAChC,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU,IAAI,UAAU,CAAC,mBAAmB;oBACpE,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,UAAU;oBAC9C,oBAAoB,EAAE,IAAI,CAAC,MAAM,CAAC,oBAAoB,IAAI,KAAK;iBAChE;aACF;SACF,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,OAAO,GAA2B;gBACtC,MAAM,EAAE,qCAAqC;gBAC7C,cAAc,EAAE,kBAAkB;aACnC,CAAC;YAEF,MAAM,QAAQ,GAAG,MAAM,KAAK,CAC1B,GAAG,UAAU,CAAC,QAAQ,GAAG,UAAU,CAAC,SAAS,CAAC,MAAM,EAAE,EACtD;gBACE,MAAM,EAAE,MAAM;gBACd,OAAO;gBACP,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC;gBACnC,MAAM;aACP,CACF,CAAC;YAEF,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACxC,MAAM,IAAI,KAAK,CAAC,iBAAiB,QAAQ,CAAC,MAAM,MAAM,SAAS,EAAE,CAAC,CAAC;YACrE,CAAC;YAED,MAAM,YAAY,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YAE3C,qBAAqB;YACrB,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACvC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,IAAI,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;oBAC9B,IAAI,CAAC;wBACH,MAAM,IAAI,GAAsB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;wBAC9D,IACE,IAAI,CAAC,MAAM;4BACX,IAAI,CAAC,MAAM,CAAC,OAAO;4BACnB,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAC9B,CAAC;4BACD,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;4BAC5C,OAAO;gCACL,UAAU,EAAE,OAAO;gCACnB,aAAa,EAAE,OAAO;6BACvB,CAAC;wBACJ,CAAC;oBACH,CAAC;oBAAC,OAAO,EAAE,EAAE,CAAC;wBACZ,sBAAsB;oBACxB,CAAC;gBACH,CAAC;YACH,CAAC;YAED,OAAO;gBACL,UAAU,EAAE,wDAAwD;gBACpE,aAAa,EAAE,mBAAmB;aACnC,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAChB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACzD,OAAO;gBACL,UAAU,EAAE,gCAAgC,YAAY,EAAE;gBAC1D,aAAa,EAAE,UAAU,YAAY,EAAE;gBACvC,KAAK,EAAE;oBACL,OAAO,EAAE,YAAY;oBACrB,IAAI,EAAE,aAAa,CAAC,iBAAiB;iBACtC;aACF,CAAC;QACJ,CAAC;IACH,CAAC;CACF"}
|
|
@@ -9,7 +9,7 @@ import { Config } from '../config/config.js';
|
|
|
9
9
|
/**
|
|
10
10
|
* Parameters for the WebFetch tool
|
|
11
11
|
*/
|
|
12
|
-
export interface
|
|
12
|
+
export interface GoogleWebFetchToolParams {
|
|
13
13
|
/**
|
|
14
14
|
* The prompt containing URL(s) (up to 20) and instructions for processing their content.
|
|
15
15
|
*/
|
|
@@ -18,10 +18,10 @@ export interface WebFetchToolParams {
|
|
|
18
18
|
/**
|
|
19
19
|
* Implementation of the WebFetch tool logic
|
|
20
20
|
*/
|
|
21
|
-
export declare class
|
|
21
|
+
export declare class GoogleWebFetchTool extends BaseDeclarativeTool<GoogleWebFetchToolParams, ToolResult> {
|
|
22
22
|
private readonly config;
|
|
23
23
|
static readonly Name: string;
|
|
24
24
|
constructor(config: Config, messageBus?: MessageBus);
|
|
25
|
-
protected validateToolParamValues(params:
|
|
26
|
-
protected createInvocation(params:
|
|
25
|
+
protected validateToolParamValues(params: GoogleWebFetchToolParams): string | null;
|
|
26
|
+
protected createInvocation(params: GoogleWebFetchToolParams, messageBus?: MessageBus): ToolInvocation<GoogleWebFetchToolParams, ToolResult>;
|
|
27
27
|
}
|
|
@@ -18,27 +18,34 @@ function extractUrls(text) {
|
|
|
18
18
|
const urlRegex = /(https?:\/\/[^\s]+)/g;
|
|
19
19
|
return text.match(urlRegex) || [];
|
|
20
20
|
}
|
|
21
|
-
class
|
|
21
|
+
class GoogleWebFetchToolInvocation extends BaseToolInvocation {
|
|
22
22
|
config;
|
|
23
23
|
constructor(config, params, messageBus) {
|
|
24
24
|
super(params, messageBus);
|
|
25
25
|
this.config = config;
|
|
26
26
|
}
|
|
27
27
|
getToolName() {
|
|
28
|
-
return
|
|
28
|
+
return GoogleWebFetchTool.Name;
|
|
29
29
|
}
|
|
30
30
|
async executeFallback(_signal) {
|
|
31
31
|
const urls = extractUrls(this.params.prompt);
|
|
32
32
|
// For now, we only support one URL for fallback
|
|
33
33
|
let url = urls[0];
|
|
34
34
|
// Convert GitHub blob URL to raw URL
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
35
|
+
// Convert GitHub blob URL to raw URL
|
|
36
|
+
try {
|
|
37
|
+
const urlObj = new URL(url);
|
|
38
|
+
if (urlObj.hostname === 'github.com' && url.includes('/blob/')) {
|
|
39
|
+
url = url
|
|
40
|
+
.replace('github.com', 'raw.githubusercontent.com')
|
|
41
|
+
.replace('/blob/', '/');
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
catch (_) {
|
|
45
|
+
// Ignore invalid URLs, they will be caught by fetchWithTimeout
|
|
39
46
|
}
|
|
40
47
|
try {
|
|
41
|
-
const response = await fetchWithTimeout(url, URL_FETCH_TIMEOUT_MS);
|
|
48
|
+
const response = await fetchWithTimeout(url, URL_FETCH_TIMEOUT_MS, _signal);
|
|
42
49
|
if (!response.ok) {
|
|
43
50
|
throw new Error(`Request failed with status code ${response.status} ${response.statusText}`);
|
|
44
51
|
}
|
|
@@ -83,10 +90,16 @@ class WebFetchToolInvocation extends BaseToolInvocation {
|
|
|
83
90
|
// Perform GitHub URL conversion here to differentiate between user-provided
|
|
84
91
|
// URL and the actual URL to be fetched.
|
|
85
92
|
const urls = extractUrls(this.params.prompt).map((url) => {
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
93
|
+
try {
|
|
94
|
+
const urlObj = new URL(url);
|
|
95
|
+
if (urlObj.hostname === 'github.com' && url.includes('/blob/')) {
|
|
96
|
+
return url
|
|
97
|
+
.replace('github.com', 'raw.githubusercontent.com')
|
|
98
|
+
.replace('/blob/', '/');
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
catch (_) {
|
|
102
|
+
// Ignore invalid URLs
|
|
90
103
|
}
|
|
91
104
|
return url;
|
|
92
105
|
});
|
|
@@ -250,11 +263,11 @@ ${sourceListFormatted.join('\n')}`;
|
|
|
250
263
|
/**
|
|
251
264
|
* Implementation of the WebFetch tool logic
|
|
252
265
|
*/
|
|
253
|
-
export class
|
|
266
|
+
export class GoogleWebFetchTool extends BaseDeclarativeTool {
|
|
254
267
|
config;
|
|
255
|
-
static Name = '
|
|
268
|
+
static Name = 'google_web_fetch';
|
|
256
269
|
constructor(config, messageBus) {
|
|
257
|
-
super(
|
|
270
|
+
super(GoogleWebFetchTool.Name, 'GoogleWebFetch', "Processes content from URL(s), including local and private network addresses (e.g., localhost), embedded in a prompt. Include up to 20 URLs and instructions (e.g., summarize, extract specific data) directly in the 'prompt' parameter.", Kind.Fetch, {
|
|
258
271
|
properties: {
|
|
259
272
|
prompt: {
|
|
260
273
|
description: 'A comprehensive prompt that includes the URL(s) (up to 20) to fetch and specific instructions on how to process their content (e.g., "Summarize https://example.com/article and extract key points from https://another.com/data"). Must contain as least one URL starting with http:// or https://.',
|
|
@@ -283,7 +296,7 @@ export class WebFetchTool extends BaseDeclarativeTool {
|
|
|
283
296
|
return null;
|
|
284
297
|
}
|
|
285
298
|
createInvocation(params, messageBus) {
|
|
286
|
-
return new
|
|
299
|
+
return new GoogleWebFetchToolInvocation(this.config, params, messageBus);
|
|
287
300
|
}
|
|
288
301
|
}
|
|
289
|
-
//# sourceMappingURL=web-fetch.js.map
|
|
302
|
+
//# sourceMappingURL=google-web-fetch.js.map
|