@oevortex/ddg_search 1.0.2 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +226 -223
- package/bin/cli.js +146 -144
- package/package.json +1 -38
- package/src/index.js +82 -78
- package/src/tools/feloTool.js +85 -0
- package/src/tools/fetchUrlTool.js +118 -118
- package/src/tools/metadataTool.js +58 -58
- package/src/tools/searchTool.js +64 -64
- package/src/utils/search.js +401 -401
- package/src/utils/search_felo.js +204 -0
|
@@ -1,118 +1,118 @@
|
|
|
1
|
-
import { fetchUrlContent } from '../utils/search.js';
|
|
2
|
-
|
|
3
|
-
/**
|
|
4
|
-
* Fetch URL tool definition
|
|
5
|
-
*/
|
|
6
|
-
export const fetchUrlToolDefinition = {
|
|
7
|
-
name: 'fetch-url',
|
|
8
|
-
description: 'Fetch the content of a URL and return it as text, with options to control extraction',
|
|
9
|
-
inputSchema: {
|
|
10
|
-
type: 'object',
|
|
11
|
-
properties: {
|
|
12
|
-
url: {
|
|
13
|
-
type: 'string',
|
|
14
|
-
description: 'The URL to fetch'
|
|
15
|
-
},
|
|
16
|
-
maxLength: {
|
|
17
|
-
type: 'integer',
|
|
18
|
-
description: 'Maximum length of content to return (default: 10000)',
|
|
19
|
-
default: 10000,
|
|
20
|
-
minimum: 1000,
|
|
21
|
-
maximum: 50000
|
|
22
|
-
},
|
|
23
|
-
extractMainContent: {
|
|
24
|
-
type: 'boolean',
|
|
25
|
-
description: 'Whether to attempt to extract main content (default: true)',
|
|
26
|
-
default: true
|
|
27
|
-
},
|
|
28
|
-
includeLinks: {
|
|
29
|
-
type: 'boolean',
|
|
30
|
-
description: 'Whether to include link text (default: true)',
|
|
31
|
-
default: true
|
|
32
|
-
},
|
|
33
|
-
includeImages: {
|
|
34
|
-
type: 'boolean',
|
|
35
|
-
description: 'Whether to include image alt text (default: true)',
|
|
36
|
-
default: true
|
|
37
|
-
},
|
|
38
|
-
excludeTags: {
|
|
39
|
-
type: 'array',
|
|
40
|
-
description: 'Tags to exclude from extraction (default: script, style, etc.)',
|
|
41
|
-
items: {
|
|
42
|
-
type: 'string'
|
|
43
|
-
}
|
|
44
|
-
}
|
|
45
|
-
},
|
|
46
|
-
required: ['url']
|
|
47
|
-
},
|
|
48
|
-
annotations: {
|
|
49
|
-
title: 'Fetch URL Content',
|
|
50
|
-
readOnlyHint: true,
|
|
51
|
-
openWorldHint: true
|
|
52
|
-
}
|
|
53
|
-
};
|
|
54
|
-
|
|
55
|
-
/**
|
|
56
|
-
* Fetch URL tool handler
|
|
57
|
-
* @param {Object} params - The tool parameters
|
|
58
|
-
* @returns {Promise<Object>} - The tool result
|
|
59
|
-
*/
|
|
60
|
-
export async function fetchUrlToolHandler(params) {
|
|
61
|
-
const {
|
|
62
|
-
url,
|
|
63
|
-
maxLength = 10000,
|
|
64
|
-
extractMainContent = true,
|
|
65
|
-
includeLinks = true,
|
|
66
|
-
includeImages = true,
|
|
67
|
-
excludeTags = ['script', 'style', 'noscript', 'iframe', 'svg', 'nav', 'footer', 'header', 'aside']
|
|
68
|
-
} = params;
|
|
69
|
-
|
|
70
|
-
console.log(`Fetching content from URL: ${url} (maxLength: ${maxLength})`);
|
|
71
|
-
|
|
72
|
-
try {
|
|
73
|
-
// Fetch content with specified options
|
|
74
|
-
const content = await fetchUrlContent(url, {
|
|
75
|
-
extractMainContent,
|
|
76
|
-
includeLinks,
|
|
77
|
-
includeImages,
|
|
78
|
-
excludeTags
|
|
79
|
-
});
|
|
80
|
-
|
|
81
|
-
// Truncate content if it's too long
|
|
82
|
-
const truncatedContent = content.length > maxLength
|
|
83
|
-
? content.substring(0, maxLength) + '... [Content truncated due to length]'
|
|
84
|
-
: content;
|
|
85
|
-
|
|
86
|
-
// Add metadata about the extraction
|
|
87
|
-
const metadata = `
|
|
88
|
-
---
|
|
89
|
-
Extraction settings:
|
|
90
|
-
- URL: ${url}
|
|
91
|
-
- Main content extraction: ${extractMainContent ? 'Enabled' : 'Disabled'}
|
|
92
|
-
- Links included: ${includeLinks ? 'Yes' : 'No'}
|
|
93
|
-
- Images included: ${includeImages ? 'Yes (as alt text)' : 'No'}
|
|
94
|
-
- Content length: ${content.length} characters${content.length > maxLength ? ` (truncated to ${maxLength})` : ''}
|
|
95
|
-
---
|
|
96
|
-
`;
|
|
97
|
-
|
|
98
|
-
return {
|
|
99
|
-
content: [
|
|
100
|
-
{
|
|
101
|
-
type: 'text',
|
|
102
|
-
text: truncatedContent + metadata
|
|
103
|
-
}
|
|
104
|
-
]
|
|
105
|
-
};
|
|
106
|
-
} catch (error) {
|
|
107
|
-
console.error(`Error fetching URL ${url}:`, error);
|
|
108
|
-
return {
|
|
109
|
-
isError: true,
|
|
110
|
-
content: [
|
|
111
|
-
{
|
|
112
|
-
type: 'text',
|
|
113
|
-
text: `Error fetching URL: ${error.message}`
|
|
114
|
-
}
|
|
115
|
-
]
|
|
116
|
-
};
|
|
117
|
-
}
|
|
118
|
-
}
|
|
1
|
+
import { fetchUrlContent } from '../utils/search.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Fetch URL tool definition
|
|
5
|
+
*/
|
|
6
|
+
export const fetchUrlToolDefinition = {
|
|
7
|
+
name: 'fetch-url',
|
|
8
|
+
description: 'Fetch the content of a URL and return it as text, with options to control extraction',
|
|
9
|
+
inputSchema: {
|
|
10
|
+
type: 'object',
|
|
11
|
+
properties: {
|
|
12
|
+
url: {
|
|
13
|
+
type: 'string',
|
|
14
|
+
description: 'The URL to fetch'
|
|
15
|
+
},
|
|
16
|
+
maxLength: {
|
|
17
|
+
type: 'integer',
|
|
18
|
+
description: 'Maximum length of content to return (default: 10000)',
|
|
19
|
+
default: 10000,
|
|
20
|
+
minimum: 1000,
|
|
21
|
+
maximum: 50000
|
|
22
|
+
},
|
|
23
|
+
extractMainContent: {
|
|
24
|
+
type: 'boolean',
|
|
25
|
+
description: 'Whether to attempt to extract main content (default: true)',
|
|
26
|
+
default: true
|
|
27
|
+
},
|
|
28
|
+
includeLinks: {
|
|
29
|
+
type: 'boolean',
|
|
30
|
+
description: 'Whether to include link text (default: true)',
|
|
31
|
+
default: true
|
|
32
|
+
},
|
|
33
|
+
includeImages: {
|
|
34
|
+
type: 'boolean',
|
|
35
|
+
description: 'Whether to include image alt text (default: true)',
|
|
36
|
+
default: true
|
|
37
|
+
},
|
|
38
|
+
excludeTags: {
|
|
39
|
+
type: 'array',
|
|
40
|
+
description: 'Tags to exclude from extraction (default: script, style, etc.)',
|
|
41
|
+
items: {
|
|
42
|
+
type: 'string'
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
},
|
|
46
|
+
required: ['url']
|
|
47
|
+
},
|
|
48
|
+
annotations: {
|
|
49
|
+
title: 'Fetch URL Content',
|
|
50
|
+
readOnlyHint: true,
|
|
51
|
+
openWorldHint: true
|
|
52
|
+
}
|
|
53
|
+
};
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Fetch URL tool handler
|
|
57
|
+
* @param {Object} params - The tool parameters
|
|
58
|
+
* @returns {Promise<Object>} - The tool result
|
|
59
|
+
*/
|
|
60
|
+
export async function fetchUrlToolHandler(params) {
|
|
61
|
+
const {
|
|
62
|
+
url,
|
|
63
|
+
maxLength = 10000,
|
|
64
|
+
extractMainContent = true,
|
|
65
|
+
includeLinks = true,
|
|
66
|
+
includeImages = true,
|
|
67
|
+
excludeTags = ['script', 'style', 'noscript', 'iframe', 'svg', 'nav', 'footer', 'header', 'aside']
|
|
68
|
+
} = params;
|
|
69
|
+
|
|
70
|
+
console.log(`Fetching content from URL: ${url} (maxLength: ${maxLength})`);
|
|
71
|
+
|
|
72
|
+
try {
|
|
73
|
+
// Fetch content with specified options
|
|
74
|
+
const content = await fetchUrlContent(url, {
|
|
75
|
+
extractMainContent,
|
|
76
|
+
includeLinks,
|
|
77
|
+
includeImages,
|
|
78
|
+
excludeTags
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
// Truncate content if it's too long
|
|
82
|
+
const truncatedContent = content.length > maxLength
|
|
83
|
+
? content.substring(0, maxLength) + '... [Content truncated due to length]'
|
|
84
|
+
: content;
|
|
85
|
+
|
|
86
|
+
// Add metadata about the extraction
|
|
87
|
+
const metadata = `
|
|
88
|
+
---
|
|
89
|
+
Extraction settings:
|
|
90
|
+
- URL: ${url}
|
|
91
|
+
- Main content extraction: ${extractMainContent ? 'Enabled' : 'Disabled'}
|
|
92
|
+
- Links included: ${includeLinks ? 'Yes' : 'No'}
|
|
93
|
+
- Images included: ${includeImages ? 'Yes (as alt text)' : 'No'}
|
|
94
|
+
- Content length: ${content.length} characters${content.length > maxLength ? ` (truncated to ${maxLength})` : ''}
|
|
95
|
+
---
|
|
96
|
+
`;
|
|
97
|
+
|
|
98
|
+
return {
|
|
99
|
+
content: [
|
|
100
|
+
{
|
|
101
|
+
type: 'text',
|
|
102
|
+
text: truncatedContent + metadata
|
|
103
|
+
}
|
|
104
|
+
]
|
|
105
|
+
};
|
|
106
|
+
} catch (error) {
|
|
107
|
+
console.error(`Error fetching URL ${url}:`, error);
|
|
108
|
+
return {
|
|
109
|
+
isError: true,
|
|
110
|
+
content: [
|
|
111
|
+
{
|
|
112
|
+
type: 'text',
|
|
113
|
+
text: `Error fetching URL: ${error.message}`
|
|
114
|
+
}
|
|
115
|
+
]
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
}
|
|
@@ -1,58 +1,58 @@
|
|
|
1
|
-
import { extractUrlMetadata } from '../utils/search.js';
|
|
2
|
-
|
|
3
|
-
/**
|
|
4
|
-
* URL metadata tool definition
|
|
5
|
-
*/
|
|
6
|
-
export const metadataToolDefinition = {
|
|
7
|
-
name: 'url-metadata',
|
|
8
|
-
description: 'Extract metadata from a URL (title, description, etc.)',
|
|
9
|
-
inputSchema: {
|
|
10
|
-
type: 'object',
|
|
11
|
-
properties: {
|
|
12
|
-
url: {
|
|
13
|
-
type: 'string',
|
|
14
|
-
description: 'The URL to extract metadata from'
|
|
15
|
-
}
|
|
16
|
-
},
|
|
17
|
-
required: ['url']
|
|
18
|
-
},
|
|
19
|
-
annotations: {
|
|
20
|
-
title: 'URL Metadata',
|
|
21
|
-
readOnlyHint: true,
|
|
22
|
-
openWorldHint: true
|
|
23
|
-
}
|
|
24
|
-
};
|
|
25
|
-
|
|
26
|
-
/**
|
|
27
|
-
* URL metadata tool handler
|
|
28
|
-
* @param {Object} params - The tool parameters
|
|
29
|
-
* @returns {Promise<Object>} - The tool result
|
|
30
|
-
*/
|
|
31
|
-
export async function metadataToolHandler(params) {
|
|
32
|
-
const { url } = params;
|
|
33
|
-
console.log(`Extracting metadata from URL: ${url}`);
|
|
34
|
-
|
|
35
|
-
const metadata = await extractUrlMetadata(url);
|
|
36
|
-
|
|
37
|
-
// Format the metadata for display
|
|
38
|
-
const formattedMetadata = `
|
|
39
|
-
## URL Metadata for ${url}
|
|
40
|
-
|
|
41
|
-
**Title:** ${metadata.title}
|
|
42
|
-
|
|
43
|
-
**Description:** ${metadata.description}
|
|
44
|
-
|
|
45
|
-
**Image:** ${metadata.ogImage || 'None'}
|
|
46
|
-
|
|
47
|
-
**Favicon:** ${metadata.favicon || 'None'}
|
|
48
|
-
`.trim();
|
|
49
|
-
|
|
50
|
-
return {
|
|
51
|
-
content: [
|
|
52
|
-
{
|
|
53
|
-
type: 'text',
|
|
54
|
-
text: formattedMetadata
|
|
55
|
-
}
|
|
56
|
-
]
|
|
57
|
-
};
|
|
58
|
-
}
|
|
1
|
+
import { extractUrlMetadata } from '../utils/search.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* URL metadata tool definition
|
|
5
|
+
*/
|
|
6
|
+
export const metadataToolDefinition = {
|
|
7
|
+
name: 'url-metadata',
|
|
8
|
+
description: 'Extract metadata from a URL (title, description, etc.)',
|
|
9
|
+
inputSchema: {
|
|
10
|
+
type: 'object',
|
|
11
|
+
properties: {
|
|
12
|
+
url: {
|
|
13
|
+
type: 'string',
|
|
14
|
+
description: 'The URL to extract metadata from'
|
|
15
|
+
}
|
|
16
|
+
},
|
|
17
|
+
required: ['url']
|
|
18
|
+
},
|
|
19
|
+
annotations: {
|
|
20
|
+
title: 'URL Metadata',
|
|
21
|
+
readOnlyHint: true,
|
|
22
|
+
openWorldHint: true
|
|
23
|
+
}
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* URL metadata tool handler
|
|
28
|
+
* @param {Object} params - The tool parameters
|
|
29
|
+
* @returns {Promise<Object>} - The tool result
|
|
30
|
+
*/
|
|
31
|
+
export async function metadataToolHandler(params) {
|
|
32
|
+
const { url } = params;
|
|
33
|
+
console.log(`Extracting metadata from URL: ${url}`);
|
|
34
|
+
|
|
35
|
+
const metadata = await extractUrlMetadata(url);
|
|
36
|
+
|
|
37
|
+
// Format the metadata for display
|
|
38
|
+
const formattedMetadata = `
|
|
39
|
+
## URL Metadata for ${url}
|
|
40
|
+
|
|
41
|
+
**Title:** ${metadata.title}
|
|
42
|
+
|
|
43
|
+
**Description:** ${metadata.description}
|
|
44
|
+
|
|
45
|
+
**Image:** ${metadata.ogImage || 'None'}
|
|
46
|
+
|
|
47
|
+
**Favicon:** ${metadata.favicon || 'None'}
|
|
48
|
+
`.trim();
|
|
49
|
+
|
|
50
|
+
return {
|
|
51
|
+
content: [
|
|
52
|
+
{
|
|
53
|
+
type: 'text',
|
|
54
|
+
text: formattedMetadata
|
|
55
|
+
}
|
|
56
|
+
]
|
|
57
|
+
};
|
|
58
|
+
}
|
package/src/tools/searchTool.js
CHANGED
|
@@ -1,64 +1,64 @@
|
|
|
1
|
-
import { searchDuckDuckGo } from '../utils/search.js';
|
|
2
|
-
|
|
3
|
-
/**
|
|
4
|
-
* Web search tool definition
|
|
5
|
-
*/
|
|
6
|
-
export const searchToolDefinition = {
|
|
7
|
-
name: 'web-search',
|
|
8
|
-
description: 'Search the web using DuckDuckGo and return results',
|
|
9
|
-
inputSchema: {
|
|
10
|
-
type: 'object',
|
|
11
|
-
properties: {
|
|
12
|
-
query: {
|
|
13
|
-
type: 'string',
|
|
14
|
-
description: 'The search query'
|
|
15
|
-
},
|
|
16
|
-
page: {
|
|
17
|
-
type: 'integer',
|
|
18
|
-
description: 'Page number (default: 1)',
|
|
19
|
-
default: 1,
|
|
20
|
-
minimum: 1
|
|
21
|
-
},
|
|
22
|
-
numResults: {
|
|
23
|
-
type: 'integer',
|
|
24
|
-
description: 'Number of results to return (default: 10)',
|
|
25
|
-
default: 10,
|
|
26
|
-
minimum: 1,
|
|
27
|
-
maximum: 20
|
|
28
|
-
}
|
|
29
|
-
},
|
|
30
|
-
required: ['query']
|
|
31
|
-
},
|
|
32
|
-
annotations: {
|
|
33
|
-
title: 'Web Search',
|
|
34
|
-
readOnlyHint: true,
|
|
35
|
-
openWorldHint: true
|
|
36
|
-
}
|
|
37
|
-
};
|
|
38
|
-
|
|
39
|
-
/**
|
|
40
|
-
* Web search tool handler
|
|
41
|
-
* @param {Object} params - The tool parameters
|
|
42
|
-
* @returns {Promise<Object>} - The tool result
|
|
43
|
-
*/
|
|
44
|
-
export async function searchToolHandler(params) {
|
|
45
|
-
const { query, page = 1, numResults = 10 } = params;
|
|
46
|
-
console.log(`Searching for: ${query} (page ${page}, ${numResults} results)`);
|
|
47
|
-
|
|
48
|
-
const results = await searchDuckDuckGo(query, page, numResults);
|
|
49
|
-
console.log(`Found ${results.length} results`);
|
|
50
|
-
|
|
51
|
-
// Format the results for display
|
|
52
|
-
const formattedResults = results.map((result, index) =>
|
|
53
|
-
`${index + 1}. [${result.title}](${result.url})\n ${result.snippet}`
|
|
54
|
-
).join('\n\n');
|
|
55
|
-
|
|
56
|
-
return {
|
|
57
|
-
content: [
|
|
58
|
-
{
|
|
59
|
-
type: 'text',
|
|
60
|
-
text: formattedResults || 'No results found.'
|
|
61
|
-
}
|
|
62
|
-
]
|
|
63
|
-
};
|
|
64
|
-
}
|
|
1
|
+
import { searchDuckDuckGo } from '../utils/search.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Web search tool definition
|
|
5
|
+
*/
|
|
6
|
+
export const searchToolDefinition = {
|
|
7
|
+
name: 'web-search',
|
|
8
|
+
description: 'Search the web using DuckDuckGo and return results',
|
|
9
|
+
inputSchema: {
|
|
10
|
+
type: 'object',
|
|
11
|
+
properties: {
|
|
12
|
+
query: {
|
|
13
|
+
type: 'string',
|
|
14
|
+
description: 'The search query'
|
|
15
|
+
},
|
|
16
|
+
page: {
|
|
17
|
+
type: 'integer',
|
|
18
|
+
description: 'Page number (default: 1)',
|
|
19
|
+
default: 1,
|
|
20
|
+
minimum: 1
|
|
21
|
+
},
|
|
22
|
+
numResults: {
|
|
23
|
+
type: 'integer',
|
|
24
|
+
description: 'Number of results to return (default: 10)',
|
|
25
|
+
default: 10,
|
|
26
|
+
minimum: 1,
|
|
27
|
+
maximum: 20
|
|
28
|
+
}
|
|
29
|
+
},
|
|
30
|
+
required: ['query']
|
|
31
|
+
},
|
|
32
|
+
annotations: {
|
|
33
|
+
title: 'Web Search',
|
|
34
|
+
readOnlyHint: true,
|
|
35
|
+
openWorldHint: true
|
|
36
|
+
}
|
|
37
|
+
};
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Web search tool handler
|
|
41
|
+
* @param {Object} params - The tool parameters
|
|
42
|
+
* @returns {Promise<Object>} - The tool result
|
|
43
|
+
*/
|
|
44
|
+
export async function searchToolHandler(params) {
|
|
45
|
+
const { query, page = 1, numResults = 10 } = params;
|
|
46
|
+
console.log(`Searching for: ${query} (page ${page}, ${numResults} results)`);
|
|
47
|
+
|
|
48
|
+
const results = await searchDuckDuckGo(query, page, numResults);
|
|
49
|
+
console.log(`Found ${results.length} results`);
|
|
50
|
+
|
|
51
|
+
// Format the results for display
|
|
52
|
+
const formattedResults = results.map((result, index) =>
|
|
53
|
+
`${index + 1}. [${result.title}](${result.url})\n ${result.snippet}`
|
|
54
|
+
).join('\n\n');
|
|
55
|
+
|
|
56
|
+
return {
|
|
57
|
+
content: [
|
|
58
|
+
{
|
|
59
|
+
type: 'text',
|
|
60
|
+
text: formattedResults || 'No results found.'
|
|
61
|
+
}
|
|
62
|
+
]
|
|
63
|
+
};
|
|
64
|
+
}
|