mcp-researchpowerpack 3.9.4 → 4.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +116 -208
- package/dist/index.js +280 -337
- package/dist/index.js.map +7 -1
- package/dist/mcp-use.json +7 -0
- package/dist/src/clients/reddit.js +278 -0
- package/dist/src/clients/reddit.js.map +7 -0
- package/dist/src/clients/scraper.js +326 -0
- package/dist/src/clients/scraper.js.map +7 -0
- package/dist/src/clients/search.js +217 -0
- package/dist/src/clients/search.js.map +7 -0
- package/dist/src/config/index.js +138 -0
- package/dist/src/config/index.js.map +7 -0
- package/dist/src/prompts/deep-research.js +41 -0
- package/dist/src/prompts/deep-research.js.map +7 -0
- package/dist/src/prompts/reddit-sentiment.js +47 -0
- package/dist/src/prompts/reddit-sentiment.js.map +7 -0
- package/dist/src/schemas/reddit.js +21 -0
- package/dist/src/schemas/reddit.js.map +7 -0
- package/dist/src/schemas/scrape-links.js +26 -0
- package/dist/src/schemas/scrape-links.js.map +7 -0
- package/dist/src/schemas/start-research.js +17 -0
- package/dist/src/schemas/start-research.js.map +7 -0
- package/dist/src/schemas/web-search.js +53 -0
- package/dist/src/schemas/web-search.js.map +7 -0
- package/dist/src/services/llm-processor.js +675 -0
- package/dist/src/services/llm-processor.js.map +7 -0
- package/dist/src/services/markdown-cleaner.js +62 -0
- package/dist/src/services/markdown-cleaner.js.map +7 -0
- package/dist/src/services/workflow-state.js +116 -0
- package/dist/src/services/workflow-state.js.map +7 -0
- package/dist/src/tools/mcp-helpers.js +50 -0
- package/dist/src/tools/mcp-helpers.js.map +7 -0
- package/dist/src/tools/reddit.js +277 -0
- package/dist/src/tools/reddit.js.map +7 -0
- package/dist/src/tools/registry.js +18 -0
- package/dist/src/tools/registry.js.map +7 -0
- package/dist/src/tools/scrape.js +334 -0
- package/dist/src/tools/scrape.js.map +7 -0
- package/dist/src/tools/search.js +423 -0
- package/dist/src/tools/search.js.map +7 -0
- package/dist/src/tools/start-research.js +199 -0
- package/dist/src/tools/start-research.js.map +7 -0
- package/dist/src/tools/utils.js +29 -0
- package/dist/src/tools/utils.js.map +7 -0
- package/dist/src/utils/bootstrap-guard.js +27 -0
- package/dist/src/utils/bootstrap-guard.js.map +7 -0
- package/dist/src/utils/concurrency.js +62 -0
- package/dist/src/utils/concurrency.js.map +7 -0
- package/dist/src/utils/content-extractor.js +61 -0
- package/dist/src/utils/content-extractor.js.map +7 -0
- package/dist/src/utils/errors.js +211 -0
- package/dist/src/utils/errors.js.map +7 -0
- package/dist/src/utils/logger.js +25 -0
- package/dist/src/utils/logger.js.map +7 -0
- package/dist/src/utils/markdown-formatter.js +15 -0
- package/dist/src/utils/markdown-formatter.js.map +7 -0
- package/dist/src/utils/reddit-keyword-guard.js +29 -0
- package/dist/src/utils/reddit-keyword-guard.js.map +7 -0
- package/dist/src/utils/response.js +81 -0
- package/dist/src/utils/response.js.map +7 -0
- package/dist/src/utils/retry.js +13 -0
- package/dist/src/utils/retry.js.map +7 -0
- package/dist/src/utils/sanitize.js +10 -0
- package/dist/src/utils/sanitize.js.map +7 -0
- package/dist/src/utils/source-type.js +41 -0
- package/dist/src/utils/source-type.js.map +7 -0
- package/dist/src/utils/url-aggregator.js +227 -0
- package/dist/src/utils/url-aggregator.js.map +7 -0
- package/dist/src/utils/workflow-key.js +14 -0
- package/dist/src/utils/workflow-key.js.map +7 -0
- package/dist/src/version.js +32 -0
- package/dist/src/version.js.map +7 -0
- package/package.json +33 -28
- package/dist/clients/reddit.d.ts +0 -69
- package/dist/clients/reddit.d.ts.map +0 -1
- package/dist/clients/reddit.js +0 -369
- package/dist/clients/reddit.js.map +0 -1
- package/dist/clients/research.d.ts +0 -67
- package/dist/clients/research.d.ts.map +0 -1
- package/dist/clients/research.js +0 -290
- package/dist/clients/research.js.map +0 -1
- package/dist/clients/scraper.d.ts +0 -72
- package/dist/clients/scraper.d.ts.map +0 -1
- package/dist/clients/scraper.js +0 -338
- package/dist/clients/scraper.js.map +0 -1
- package/dist/clients/search.d.ts +0 -57
- package/dist/clients/search.d.ts.map +0 -1
- package/dist/clients/search.js +0 -223
- package/dist/clients/search.js.map +0 -1
- package/dist/config/index.d.ts +0 -78
- package/dist/config/index.d.ts.map +0 -1
- package/dist/config/index.js +0 -201
- package/dist/config/index.js.map +0 -1
- package/dist/config/loader.d.ts +0 -40
- package/dist/config/loader.d.ts.map +0 -1
- package/dist/config/loader.js +0 -322
- package/dist/config/loader.js.map +0 -1
- package/dist/config/types.d.ts +0 -81
- package/dist/config/types.d.ts.map +0 -1
- package/dist/config/types.js +0 -6
- package/dist/config/types.js.map +0 -1
- package/dist/config/yaml/tools.yaml +0 -146
- package/dist/index.d.ts +0 -7
- package/dist/index.d.ts.map +0 -1
- package/dist/schemas/deep-research.d.ts +0 -64
- package/dist/schemas/deep-research.d.ts.map +0 -1
- package/dist/schemas/deep-research.js +0 -224
- package/dist/schemas/deep-research.js.map +0 -1
- package/dist/schemas/scrape-links.d.ts +0 -32
- package/dist/schemas/scrape-links.d.ts.map +0 -1
- package/dist/schemas/scrape-links.js +0 -34
- package/dist/schemas/scrape-links.js.map +0 -1
- package/dist/schemas/web-search.d.ts +0 -22
- package/dist/schemas/web-search.d.ts.map +0 -1
- package/dist/schemas/web-search.js +0 -21
- package/dist/schemas/web-search.js.map +0 -1
- package/dist/services/file-attachment.d.ts +0 -30
- package/dist/services/file-attachment.d.ts.map +0 -1
- package/dist/services/file-attachment.js +0 -207
- package/dist/services/file-attachment.js.map +0 -1
- package/dist/services/llm-processor.d.ts +0 -29
- package/dist/services/llm-processor.d.ts.map +0 -1
- package/dist/services/llm-processor.js +0 -244
- package/dist/services/llm-processor.js.map +0 -1
- package/dist/services/markdown-cleaner.d.ts +0 -8
- package/dist/services/markdown-cleaner.d.ts.map +0 -1
- package/dist/services/markdown-cleaner.js +0 -74
- package/dist/services/markdown-cleaner.js.map +0 -1
- package/dist/tools/definitions.d.ts +0 -16
- package/dist/tools/definitions.d.ts.map +0 -1
- package/dist/tools/definitions.js +0 -17
- package/dist/tools/definitions.js.map +0 -1
- package/dist/tools/reddit.d.ts +0 -14
- package/dist/tools/reddit.d.ts.map +0 -1
- package/dist/tools/reddit.js +0 -265
- package/dist/tools/reddit.js.map +0 -1
- package/dist/tools/registry.d.ts +0 -71
- package/dist/tools/registry.d.ts.map +0 -1
- package/dist/tools/registry.js +0 -252
- package/dist/tools/registry.js.map +0 -1
- package/dist/tools/research.d.ts +0 -14
- package/dist/tools/research.d.ts.map +0 -1
- package/dist/tools/research.js +0 -196
- package/dist/tools/research.js.map +0 -1
- package/dist/tools/scrape.d.ts +0 -14
- package/dist/tools/scrape.d.ts.map +0 -1
- package/dist/tools/scrape.js +0 -234
- package/dist/tools/scrape.js.map +0 -1
- package/dist/tools/search.d.ts +0 -10
- package/dist/tools/search.d.ts.map +0 -1
- package/dist/tools/search.js +0 -158
- package/dist/tools/search.js.map +0 -1
- package/dist/tools/utils.d.ts +0 -105
- package/dist/tools/utils.d.ts.map +0 -1
- package/dist/tools/utils.js +0 -159
- package/dist/tools/utils.js.map +0 -1
- package/dist/utils/concurrency.d.ts +0 -28
- package/dist/utils/concurrency.d.ts.map +0 -1
- package/dist/utils/concurrency.js +0 -92
- package/dist/utils/concurrency.js.map +0 -1
- package/dist/utils/errors.d.ts +0 -95
- package/dist/utils/errors.d.ts.map +0 -1
- package/dist/utils/errors.js +0 -390
- package/dist/utils/errors.js.map +0 -1
- package/dist/utils/logger.d.ts +0 -39
- package/dist/utils/logger.d.ts.map +0 -1
- package/dist/utils/logger.js +0 -57
- package/dist/utils/logger.js.map +0 -1
- package/dist/utils/markdown-formatter.d.ts +0 -5
- package/dist/utils/markdown-formatter.d.ts.map +0 -1
- package/dist/utils/markdown-formatter.js +0 -15
- package/dist/utils/markdown-formatter.js.map +0 -1
- package/dist/utils/response.d.ts +0 -93
- package/dist/utils/response.d.ts.map +0 -1
- package/dist/utils/response.js +0 -170
- package/dist/utils/response.js.map +0 -1
- package/dist/utils/retry.d.ts +0 -43
- package/dist/utils/retry.d.ts.map +0 -1
- package/dist/utils/retry.js +0 -57
- package/dist/utils/retry.js.map +0 -1
- package/dist/utils/url-aggregator.d.ts +0 -90
- package/dist/utils/url-aggregator.d.ts.map +0 -1
- package/dist/utils/url-aggregator.js +0 -538
- package/dist/utils/url-aggregator.js.map +0 -1
- package/dist/version.d.ts +0 -29
- package/dist/version.d.ts.map +0 -1
- package/dist/version.js +0 -55
- package/dist/version.js.map +0 -1
|
@@ -0,0 +1,334 @@
|
|
|
1
|
+
import { SCRAPER, CONCURRENCY, getCapabilities, getMissingEnvMessage } from "../config/index.js";
|
|
2
|
+
import {
|
|
3
|
+
scrapeLinksOutputSchema,
|
|
4
|
+
scrapeLinksParamsSchema
|
|
5
|
+
} from "../schemas/scrape-links.js";
|
|
6
|
+
import { ScraperClient } from "../clients/scraper.js";
|
|
7
|
+
import { MarkdownCleaner } from "../services/markdown-cleaner.js";
|
|
8
|
+
import { createLLMProcessor, processContentWithLLM } from "../services/llm-processor.js";
|
|
9
|
+
import { removeMetaTags } from "../utils/markdown-formatter.js";
|
|
10
|
+
import { extractReadableContent } from "../utils/content-extractor.js";
|
|
11
|
+
import { classifyError } from "../utils/errors.js";
|
|
12
|
+
import { pMap } from "../utils/concurrency.js";
|
|
13
|
+
import {
|
|
14
|
+
mcpLog,
|
|
15
|
+
formatSuccess,
|
|
16
|
+
formatError,
|
|
17
|
+
formatBatchHeader,
|
|
18
|
+
formatDuration,
|
|
19
|
+
TOKEN_BUDGETS,
|
|
20
|
+
calculateTokenAllocation
|
|
21
|
+
} from "./utils.js";
|
|
22
|
+
import {
|
|
23
|
+
createToolReporter,
|
|
24
|
+
NOOP_REPORTER,
|
|
25
|
+
toolFailure,
|
|
26
|
+
toolSuccess,
|
|
27
|
+
toToolResponse
|
|
28
|
+
} from "./mcp-helpers.js";
|
|
29
|
+
import { requireBootstrap } from "../utils/bootstrap-guard.js";
|
|
30
|
+
const markdownCleaner = new MarkdownCleaner();
|
|
31
|
+
function getExtractionPrefix() {
|
|
32
|
+
return SCRAPER.EXTRACTION_PREFIX;
|
|
33
|
+
}
|
|
34
|
+
function getExtractionSuffix() {
|
|
35
|
+
return SCRAPER.EXTRACTION_SUFFIX;
|
|
36
|
+
}
|
|
37
|
+
function enhanceExtractionInstruction(instruction) {
|
|
38
|
+
const base = instruction || "Extract the main content and key information from this page.";
|
|
39
|
+
return `${getExtractionPrefix()}
|
|
40
|
+
|
|
41
|
+
${base}
|
|
42
|
+
|
|
43
|
+
${getExtractionSuffix()}`;
|
|
44
|
+
}
|
|
45
|
+
function createScrapeErrorResponse(code, message, startTime, totalUrls, retryable = false, alternatives) {
|
|
46
|
+
return toolFailure(
|
|
47
|
+
`${formatError({
|
|
48
|
+
code,
|
|
49
|
+
message,
|
|
50
|
+
retryable,
|
|
51
|
+
toolName: "scrape-links",
|
|
52
|
+
howToFix: code === "NO_URLS" ? ["Provide at least one valid URL"] : void 0,
|
|
53
|
+
alternatives
|
|
54
|
+
})}
|
|
55
|
+
|
|
56
|
+
Execution time: ${formatDuration(Date.now() - startTime)}`
|
|
57
|
+
);
|
|
58
|
+
}
|
|
59
|
+
const REDDIT_HOST = /(?:^|\.)reddit\.com$/i;
|
|
60
|
+
function isRedditUrl(url) {
|
|
61
|
+
try {
|
|
62
|
+
return REDDIT_HOST.test(new URL(url).hostname);
|
|
63
|
+
} catch {
|
|
64
|
+
return false;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
function validateAndPartitionUrls(urls) {
|
|
68
|
+
const validUrls = [];
|
|
69
|
+
const invalidUrls = [];
|
|
70
|
+
for (const url of urls) {
|
|
71
|
+
try {
|
|
72
|
+
new URL(url);
|
|
73
|
+
validUrls.push(url);
|
|
74
|
+
} catch {
|
|
75
|
+
invalidUrls.push(url);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
return { validUrls, invalidUrls };
|
|
79
|
+
}
|
|
80
|
+
function initializeScrapeClients() {
|
|
81
|
+
const client = new ScraperClient();
|
|
82
|
+
const llmProcessor = createLLMProcessor();
|
|
83
|
+
return { client, llmProcessor };
|
|
84
|
+
}
|
|
85
|
+
function processScrapeResults(results, invalidUrls) {
|
|
86
|
+
const successItems = [];
|
|
87
|
+
const failedContents = [];
|
|
88
|
+
let successful = 0;
|
|
89
|
+
let failed = 0;
|
|
90
|
+
let totalCredits = 0;
|
|
91
|
+
for (const invalidUrl of invalidUrls) {
|
|
92
|
+
failed++;
|
|
93
|
+
failedContents.push(`## ${invalidUrl}
|
|
94
|
+
|
|
95
|
+
\u274C Invalid URL format`);
|
|
96
|
+
}
|
|
97
|
+
for (let i = 0; i < results.length; i++) {
|
|
98
|
+
const result = results[i];
|
|
99
|
+
if (!result) {
|
|
100
|
+
failed++;
|
|
101
|
+
failedContents.push(`## Unknown URL
|
|
102
|
+
|
|
103
|
+
\u274C No result returned`);
|
|
104
|
+
continue;
|
|
105
|
+
}
|
|
106
|
+
mcpLog("debug", `[${i + 1}/${results.length}] Processing ${result.url}`, "scrape");
|
|
107
|
+
if (result.error || result.statusCode < 200 || result.statusCode >= 300) {
|
|
108
|
+
failed++;
|
|
109
|
+
const errorMsg = result.error?.message || result.content || `HTTP ${result.statusCode}`;
|
|
110
|
+
failedContents.push(`## ${result.url}
|
|
111
|
+
|
|
112
|
+
\u274C Failed to scrape: ${errorMsg}`);
|
|
113
|
+
mcpLog("warning", `[${i + 1}/${results.length}] Failed: ${errorMsg}`, "scrape");
|
|
114
|
+
continue;
|
|
115
|
+
}
|
|
116
|
+
successful++;
|
|
117
|
+
totalCredits += result.credits;
|
|
118
|
+
let content;
|
|
119
|
+
try {
|
|
120
|
+
const readable = extractReadableContent(result.content, result.url);
|
|
121
|
+
const sourceForCleaner = readable.extracted ? readable.content : result.content;
|
|
122
|
+
content = markdownCleaner.processContent(sourceForCleaner);
|
|
123
|
+
} catch {
|
|
124
|
+
content = result.content;
|
|
125
|
+
}
|
|
126
|
+
successItems.push({ url: result.url, content, index: i });
|
|
127
|
+
}
|
|
128
|
+
return { successItems, failedContents, metrics: { successful, failed, totalCredits } };
|
|
129
|
+
}
|
|
130
|
+
async function processItemsWithLlm(successItems, enhancedInstruction, tokensPerUrl, llmProcessor, reporter) {
|
|
131
|
+
let llmErrors = 0;
|
|
132
|
+
if (!llmProcessor || successItems.length === 0) {
|
|
133
|
+
if (!llmProcessor && successItems.length > 0) {
|
|
134
|
+
mcpLog("warning", "LLM unavailable (LLM_EXTRACTION_API_KEY not set). Returning raw scraped content.", "scrape");
|
|
135
|
+
void reporter.log("warning", "llm_extractor_unreachable: planner not configured; raw scraped content returned");
|
|
136
|
+
}
|
|
137
|
+
return { items: successItems, llmErrors, llmAttempted: 0 };
|
|
138
|
+
}
|
|
139
|
+
mcpLog("info", `Starting parallel LLM extraction for ${successItems.length} pages (concurrency: ${CONCURRENCY.LLM_EXTRACTION})`, "scrape");
|
|
140
|
+
const llmResults = await pMap(successItems, async (item) => {
|
|
141
|
+
mcpLog("debug", `LLM extracting ${item.url} (${tokensPerUrl} tokens)...`, "scrape");
|
|
142
|
+
const llmResult = await processContentWithLLM(
|
|
143
|
+
item.content,
|
|
144
|
+
{ enabled: true, extract: enhancedInstruction, max_tokens: tokensPerUrl, url: item.url },
|
|
145
|
+
llmProcessor
|
|
146
|
+
);
|
|
147
|
+
if (llmResult.processed) {
|
|
148
|
+
mcpLog("debug", `LLM extraction complete for ${item.url}`, "scrape");
|
|
149
|
+
return { ...item, content: llmResult.content };
|
|
150
|
+
}
|
|
151
|
+
llmErrors++;
|
|
152
|
+
mcpLog("warning", `LLM extraction failed for ${item.url}: ${llmResult.error || "unknown reason"}`, "scrape");
|
|
153
|
+
void reporter.log("warning", `llm_extractor_unreachable: ${item.url} \u2014 ${llmResult.error || "unknown reason"}`);
|
|
154
|
+
return item;
|
|
155
|
+
}, CONCURRENCY.LLM_EXTRACTION);
|
|
156
|
+
return { items: llmResults, llmErrors, llmAttempted: successItems.length };
|
|
157
|
+
}
|
|
158
|
+
function assembleContentEntries(successItems, failedContents) {
|
|
159
|
+
const contents = [...failedContents];
|
|
160
|
+
for (const item of successItems) {
|
|
161
|
+
let content = item.content;
|
|
162
|
+
try {
|
|
163
|
+
content = removeMetaTags(content);
|
|
164
|
+
} catch {
|
|
165
|
+
}
|
|
166
|
+
contents.push(`## ${item.url}
|
|
167
|
+
|
|
168
|
+
${content}`);
|
|
169
|
+
}
|
|
170
|
+
return contents;
|
|
171
|
+
}
|
|
172
|
+
function buildScrapeMetadata(params, metrics, tokensPerUrl, totalBatches, executionTime) {
|
|
173
|
+
return {
|
|
174
|
+
total_items: params.urls.length,
|
|
175
|
+
successful: metrics.successful,
|
|
176
|
+
failed: metrics.failed,
|
|
177
|
+
execution_time_ms: executionTime,
|
|
178
|
+
total_credits: metrics.totalCredits
|
|
179
|
+
};
|
|
180
|
+
}
|
|
181
|
+
function buildScrapeResponse(params, contents, metrics, tokensPerUrl, totalBatches, llmErrors, executionTime, llmAccounting) {
|
|
182
|
+
const llmExtras = {};
|
|
183
|
+
if (llmAccounting.llmAttempted > 0) {
|
|
184
|
+
const ok = llmAccounting.llmAttempted - llmErrors;
|
|
185
|
+
llmExtras["LLM extraction"] = `${ok}/${llmAccounting.llmAttempted} succeeded`;
|
|
186
|
+
if (!llmAccounting.llmSucceeded) {
|
|
187
|
+
llmExtras["LLM credit"] = "0 charged (no extraction produced)";
|
|
188
|
+
}
|
|
189
|
+
} else if (llmErrors > 0) {
|
|
190
|
+
llmExtras["LLM extraction failures"] = llmErrors;
|
|
191
|
+
}
|
|
192
|
+
const batchHeader = formatBatchHeader({
|
|
193
|
+
title: `Scraped Content (${params.urls.length} URLs)`,
|
|
194
|
+
totalItems: params.urls.length,
|
|
195
|
+
successful: metrics.successful,
|
|
196
|
+
failed: metrics.failed,
|
|
197
|
+
tokensPerItem: tokensPerUrl,
|
|
198
|
+
batches: totalBatches,
|
|
199
|
+
extras: {
|
|
200
|
+
"Credits used": metrics.totalCredits,
|
|
201
|
+
...llmExtras
|
|
202
|
+
}
|
|
203
|
+
});
|
|
204
|
+
const formattedContent = formatSuccess({
|
|
205
|
+
title: "Scraping Complete",
|
|
206
|
+
summary: batchHeader,
|
|
207
|
+
data: contents.join("\n\n---\n\n"),
|
|
208
|
+
metadata: {
|
|
209
|
+
"Execution time": formatDuration(executionTime),
|
|
210
|
+
"Token budget": TOKEN_BUDGETS.SCRAPER.toLocaleString()
|
|
211
|
+
}
|
|
212
|
+
});
|
|
213
|
+
const metadata = buildScrapeMetadata(params, metrics, tokensPerUrl, totalBatches, executionTime);
|
|
214
|
+
return { content: formattedContent, structuredContent: { content: formattedContent, metadata } };
|
|
215
|
+
}
|
|
216
|
+
async function handleScrapeLinks(params, reporter = NOOP_REPORTER) {
|
|
217
|
+
const startTime = Date.now();
|
|
218
|
+
if (!params.urls || params.urls.length === 0) {
|
|
219
|
+
return createScrapeErrorResponse("NO_URLS", "No URLs provided", startTime, params.urls?.length || 0);
|
|
220
|
+
}
|
|
221
|
+
const redditUrls = params.urls.filter(isRedditUrl);
|
|
222
|
+
if (redditUrls.length > 0) {
|
|
223
|
+
return createScrapeErrorResponse(
|
|
224
|
+
"UNSUPPORTED_URL_TYPE",
|
|
225
|
+
`scrape-links does not support Reddit URLs. Use get-reddit-post for: ${redditUrls.join(", ")}`,
|
|
226
|
+
startTime,
|
|
227
|
+
params.urls.length,
|
|
228
|
+
false,
|
|
229
|
+
[
|
|
230
|
+
`get-reddit-post(urls=[${redditUrls.map((u) => `"${u}"`).join(", ")}]) \u2014 fetch threaded posts and comments directly`,
|
|
231
|
+
'web-search(queries=["..."], extract="...", scope: "reddit") \u2014 find Reddit post permalinks first if these URLs were guesses'
|
|
232
|
+
]
|
|
233
|
+
);
|
|
234
|
+
}
|
|
235
|
+
const { validUrls, invalidUrls } = validateAndPartitionUrls(params.urls);
|
|
236
|
+
await reporter.log("info", `Validated ${validUrls.length} scrapeable URL(s) and ${invalidUrls.length} invalid URL(s)`);
|
|
237
|
+
if (validUrls.length === 0) {
|
|
238
|
+
return createScrapeErrorResponse("INVALID_URLS", `All ${params.urls.length} URLs are invalid`, startTime, params.urls.length, false, [
|
|
239
|
+
'web-search(queries=["topic documentation", "topic guide"], extract="relevant documentation and guides") \u2014 search for valid URLs first, then scrape the results',
|
|
240
|
+
'web-search(queries=["topic recommendations"], extract="...", scope: "reddit") \u2014 find Reddit discussion permalinks to feed get-reddit-post instead'
|
|
241
|
+
]);
|
|
242
|
+
}
|
|
243
|
+
const tokensPerUrl = calculateTokenAllocation(validUrls.length, TOKEN_BUDGETS.SCRAPER);
|
|
244
|
+
const totalBatches = Math.ceil(validUrls.length / SCRAPER.BATCH_SIZE);
|
|
245
|
+
mcpLog("info", `Starting scrape: ${validUrls.length} URL(s), ${tokensPerUrl} tokens/URL, ${totalBatches} batch(es)`, "scrape");
|
|
246
|
+
await reporter.progress(15, 100, "Preparing scraper clients");
|
|
247
|
+
let clients;
|
|
248
|
+
try {
|
|
249
|
+
clients = initializeScrapeClients();
|
|
250
|
+
} catch (error) {
|
|
251
|
+
const err = classifyError(error);
|
|
252
|
+
return createScrapeErrorResponse("CLIENT_INIT_FAILED", `Failed to initialize scraper: ${err.message}`, startTime, params.urls.length, false, [
|
|
253
|
+
'web-search(queries=["topic key findings", "topic summary", "topic overview"], extract="key findings and summary") \u2014 search for information instead of scraping',
|
|
254
|
+
'web-search(queries=["topic discussion", "topic recommendations"], extract="...", scope: "reddit") \u2014 get community insights as an alternative'
|
|
255
|
+
]);
|
|
256
|
+
}
|
|
257
|
+
const enhancedInstruction = enhanceExtractionInstruction(params.extract);
|
|
258
|
+
await reporter.progress(35, 100, "Fetching page content");
|
|
259
|
+
const results = await clients.client.scrapeMultiple(validUrls, { timeout: 60 });
|
|
260
|
+
mcpLog("info", `Scraping complete. Processing ${results.length} results...`, "scrape");
|
|
261
|
+
await reporter.log("info", `Fetched ${results.length} scrape response(s) from the provider`);
|
|
262
|
+
await reporter.progress(60, 100, "Cleaning and classifying scrape results");
|
|
263
|
+
const { successItems, failedContents, metrics } = processScrapeResults(results, invalidUrls);
|
|
264
|
+
if (successItems.length > 0) {
|
|
265
|
+
await reporter.progress(80, 100, "Running LLM extraction over scraped pages");
|
|
266
|
+
}
|
|
267
|
+
const { items: processedItems, llmErrors, llmAttempted } = await processItemsWithLlm(
|
|
268
|
+
successItems,
|
|
269
|
+
enhancedInstruction,
|
|
270
|
+
tokensPerUrl,
|
|
271
|
+
clients.llmProcessor,
|
|
272
|
+
reporter
|
|
273
|
+
);
|
|
274
|
+
const contents = assembleContentEntries(processedItems, failedContents);
|
|
275
|
+
const executionTime = Date.now() - startTime;
|
|
276
|
+
mcpLog("info", `Completed: ${metrics.successful} successful, ${metrics.failed} failed, ${metrics.totalCredits} credits used`, "scrape");
|
|
277
|
+
await reporter.log(
|
|
278
|
+
"info",
|
|
279
|
+
`Scrape completed with ${metrics.successful} success(es), ${metrics.failed} failure(s), and ${llmErrors} LLM extraction issue(s)`
|
|
280
|
+
);
|
|
281
|
+
const llmSucceeded = llmAttempted > 0 && llmErrors < llmAttempted;
|
|
282
|
+
const result = buildScrapeResponse(
|
|
283
|
+
params,
|
|
284
|
+
contents,
|
|
285
|
+
metrics,
|
|
286
|
+
tokensPerUrl,
|
|
287
|
+
totalBatches,
|
|
288
|
+
llmErrors,
|
|
289
|
+
executionTime,
|
|
290
|
+
{ llmAttempted, llmSucceeded }
|
|
291
|
+
);
|
|
292
|
+
if (metrics.successful === 0 && metrics.failed > 0) {
|
|
293
|
+
return toolFailure(result.content);
|
|
294
|
+
}
|
|
295
|
+
return toolSuccess(result.content, result.structuredContent);
|
|
296
|
+
}
|
|
297
|
+
function registerScrapeLinksTool(server) {
|
|
298
|
+
server.tool(
|
|
299
|
+
{
|
|
300
|
+
name: "scrape-links",
|
|
301
|
+
title: "Scrape Links",
|
|
302
|
+
description: "Scrape many web pages in parallel and run structured LLM extraction on each (no hard cap). Per-page output: `## Source` (URL + detected page type + date), `## Matches` (verbatim-preserved facts), `## Not found` (explicit gaps the page did not answer \u2014 kills hallucination), `## Follow-up signals` (new terms and referenced-but-unscraped URLs that feed the next research loop), optional `## Contradictions` and `## Truncation` footers. Extraction behavior adapts per page type (docs, github-thread, reddit, marketing, cve, paper, announcement, qa, blog, changelog, release-notes). Use the `extract` field to describe the shape of what you want, separated by `|`.",
|
|
303
|
+
schema: scrapeLinksParamsSchema,
|
|
304
|
+
outputSchema: scrapeLinksOutputSchema,
|
|
305
|
+
annotations: {
|
|
306
|
+
readOnlyHint: true,
|
|
307
|
+
idempotentHint: true,
|
|
308
|
+
destructiveHint: false,
|
|
309
|
+
openWorldHint: true,
|
|
310
|
+
// See contract-fixes/03 — non-standard precondition hint.
|
|
311
|
+
...{ experimental: { requires: ["start-research"] } }
|
|
312
|
+
},
|
|
313
|
+
_meta: { requires: ["start-research"] }
|
|
314
|
+
},
|
|
315
|
+
async (args, ctx) => {
|
|
316
|
+
if (!getCapabilities().scraping) {
|
|
317
|
+
return toToolResponse(toolFailure(getMissingEnvMessage("scraping")));
|
|
318
|
+
}
|
|
319
|
+
const guard = await requireBootstrap(ctx);
|
|
320
|
+
if (guard) {
|
|
321
|
+
return guard;
|
|
322
|
+
}
|
|
323
|
+
const reporter = createToolReporter(ctx, "scrape-links");
|
|
324
|
+
const result = await handleScrapeLinks(args, reporter);
|
|
325
|
+
await reporter.progress(100, 100, result.isError ? "Scrape failed" : "Scrape complete");
|
|
326
|
+
return toToolResponse(result);
|
|
327
|
+
}
|
|
328
|
+
);
|
|
329
|
+
}
|
|
330
|
+
export {
|
|
331
|
+
handleScrapeLinks,
|
|
332
|
+
registerScrapeLinksTool
|
|
333
|
+
};
|
|
334
|
+
//# sourceMappingURL=scrape.js.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../../../src/tools/scrape.ts"],
|
|
4
|
+
"sourcesContent": ["/**\n * Scrape Links Tool Handler\n * Implements robust error handling that NEVER crashes the MCP server\n */\n\nimport type { MCPServer } from 'mcp-use/server';\n\nimport { SCRAPER, CONCURRENCY, getCapabilities, getMissingEnvMessage } from '../config/index.js';\nimport {\n scrapeLinksOutputSchema,\n scrapeLinksParamsSchema,\n type ScrapeLinksParams,\n type ScrapeLinksOutput,\n} from '../schemas/scrape-links.js';\nimport { ScraperClient } from '../clients/scraper.js';\nimport { MarkdownCleaner } from '../services/markdown-cleaner.js';\nimport { createLLMProcessor, processContentWithLLM } from '../services/llm-processor.js';\nimport { removeMetaTags } from '../utils/markdown-formatter.js';\nimport { extractReadableContent } from '../utils/content-extractor.js';\nimport { classifyError } from '../utils/errors.js';\nimport { pMap } from '../utils/concurrency.js';\nimport {\n mcpLog,\n formatSuccess,\n formatError,\n formatBatchHeader,\n formatDuration,\n TOKEN_BUDGETS,\n calculateTokenAllocation,\n} from './utils.js';\nimport {\n createToolReporter,\n NOOP_REPORTER,\n toolFailure,\n toolSuccess,\n toToolResponse,\n type ToolExecutionResult,\n type ToolReporter,\n} from './mcp-helpers.js';\nimport { requireBootstrap } from '../utils/bootstrap-guard.js';\n\n// Module-level singleton - MarkdownCleaner is stateless\nconst markdownCleaner = new MarkdownCleaner();\n\n// Extraction prefix/suffix are kept in runtime config to avoid YAML indirection.\nfunction getExtractionPrefix(): string {\n return SCRAPER.EXTRACTION_PREFIX;\n}\n\nfunction getExtractionSuffix(): string {\n return SCRAPER.EXTRACTION_SUFFIX;\n}\n\nfunction enhanceExtractionInstruction(instruction: string | undefined): string {\n const base = instruction || 'Extract the main content and key information from this page.';\n return `${getExtractionPrefix()}\\n\\n${base}\\n\\n${getExtractionSuffix()}`;\n}\n\n// --- Internal types for decomposed helpers ---\n\ninterface ProcessedResult {\n url: string;\n content: string;\n index: number;\n}\n\ninterface ScrapeMetrics {\n successful: number;\n failed: number;\n totalCredits: number;\n}\n\ninterface ScrapePhaseResult {\n successItems: ProcessedResult[];\n failedContents: string[];\n metrics: ScrapeMetrics;\n}\n\ninterface ScrapeClients {\n client: ScraperClient;\n llmProcessor: ReturnType<typeof createLLMProcessor>;\n}\n\n// --- Helpers ---\n\nfunction createScrapeErrorResponse(\n code: string,\n message: string,\n startTime: number,\n totalUrls: number,\n retryable = false,\n alternatives?: string[],\n): ToolExecutionResult<ScrapeLinksOutput> {\n return toolFailure(\n `${formatError({\n code,\n message,\n retryable,\n toolName: 'scrape-links',\n howToFix: code === 'NO_URLS' ? ['Provide at least one valid URL'] : undefined,\n alternatives,\n })}\\n\\nExecution time: ${formatDuration(Date.now() - startTime)}`,\n );\n}\n\n/** Reddit subdomains that should be routed to get-reddit-post instead. */\nconst REDDIT_HOST = /(?:^|\\.)reddit\\.com$/i;\n\nfunction isRedditUrl(url: string): boolean {\n try {\n return REDDIT_HOST.test(new URL(url).hostname);\n } catch {\n return false;\n }\n}\n\nfunction validateAndPartitionUrls(urls: string[]): { validUrls: string[]; invalidUrls: string[] } {\n const validUrls: string[] = [];\n const invalidUrls: string[] = [];\n for (const url of urls) {\n try {\n new URL(url);\n validUrls.push(url);\n } catch {\n invalidUrls.push(url);\n }\n }\n return { validUrls, invalidUrls };\n}\n\nfunction initializeScrapeClients(): ScrapeClients {\n const client = new ScraperClient();\n const llmProcessor = createLLMProcessor();\n return { client, llmProcessor };\n}\n\nfunction processScrapeResults(\n results: Awaited<ReturnType<ScraperClient['scrapeMultiple']>>,\n invalidUrls: string[],\n): ScrapePhaseResult {\n const successItems: ProcessedResult[] = [];\n const failedContents: string[] = [];\n let successful = 0;\n let failed = 0;\n let totalCredits = 0;\n\n for (const invalidUrl of invalidUrls) {\n failed++;\n failedContents.push(`## ${invalidUrl}\\n\\n\u274C Invalid URL format`);\n }\n\n for (let i = 0; i < results.length; i++) {\n const result = results[i];\n if (!result) {\n failed++;\n failedContents.push(`## Unknown URL\\n\\n\u274C No result returned`);\n continue;\n }\n\n mcpLog('debug', `[${i + 1}/${results.length}] Processing ${result.url}`, 'scrape');\n\n if (result.error || result.statusCode < 200 || result.statusCode >= 300) {\n failed++;\n const errorMsg = result.error?.message || result.content || `HTTP ${result.statusCode}`;\n failedContents.push(`## ${result.url}\\n\\n\u274C Failed to scrape: ${errorMsg}`);\n mcpLog('warning', `[${i + 1}/${results.length}] Failed: ${errorMsg}`, 'scrape');\n continue;\n }\n\n successful++;\n totalCredits += result.credits;\n\n // Strip HTML chrome (cookie banners, nav, footer, repeated hero blocks)\n // BEFORE LLM extraction. Same pipeline applies to the raw fallback.\n // See: docs/code-review/context/02-current-tool-surface.md (E5).\n let content: string;\n try {\n const readable = extractReadableContent(result.content, result.url);\n const sourceForCleaner = readable.extracted ? readable.content : result.content;\n content = markdownCleaner.processContent(sourceForCleaner);\n } catch {\n content = result.content;\n }\n\n successItems.push({ url: result.url, content, index: i });\n }\n\n return { successItems, failedContents, metrics: { successful, failed, totalCredits } };\n}\n\nasync function processItemsWithLlm(\n successItems: ProcessedResult[],\n enhancedInstruction: string,\n tokensPerUrl: number,\n llmProcessor: ReturnType<typeof createLLMProcessor>,\n reporter: ToolReporter,\n): Promise<{ items: ProcessedResult[]; llmErrors: number; llmAttempted: number }> {\n let llmErrors = 0;\n\n if (!llmProcessor || successItems.length === 0) {\n if (!llmProcessor && successItems.length > 0) {\n mcpLog('warning', 'LLM unavailable (LLM_EXTRACTION_API_KEY not set). Returning raw scraped content.', 'scrape');\n // Surface degraded mode to the client. See:\n // mcp-revisions/llm-degradation/01-signal-degraded-via-ctx-log.md\n void reporter.log('warning', 'llm_extractor_unreachable: planner not configured; raw scraped content returned');\n }\n return { items: successItems, llmErrors, llmAttempted: 0 };\n }\n\n mcpLog('info', `Starting parallel LLM extraction for ${successItems.length} pages (concurrency: ${CONCURRENCY.LLM_EXTRACTION})`, 'scrape');\n\n const llmResults = await pMap(successItems, async (item) => {\n mcpLog('debug', `LLM extracting ${item.url} (${tokensPerUrl} tokens)...`, 'scrape');\n\n const llmResult = await processContentWithLLM(\n item.content,\n { enabled: true, extract: enhancedInstruction, max_tokens: tokensPerUrl, url: item.url },\n llmProcessor,\n );\n\n if (llmResult.processed) {\n mcpLog('debug', `LLM extraction complete for ${item.url}`, 'scrape');\n return { ...item, content: llmResult.content };\n }\n\n llmErrors++;\n mcpLog('warning', `LLM extraction failed for ${item.url}: ${llmResult.error || 'unknown reason'}`, 'scrape');\n void reporter.log('warning', `llm_extractor_unreachable: ${item.url} \u2014 ${llmResult.error || 'unknown reason'}`);\n return item;\n }, CONCURRENCY.LLM_EXTRACTION);\n\n return { items: llmResults, llmErrors, llmAttempted: successItems.length };\n}\n\nfunction assembleContentEntries(successItems: ProcessedResult[], failedContents: string[]): string[] {\n const contents = [...failedContents];\n for (const item of successItems) {\n let content = item.content;\n try {\n content = removeMetaTags(content);\n } catch {\n // Use content as-is\n }\n contents.push(`## ${item.url}\\n\\n${content}`);\n }\n return contents;\n}\n\nfunction buildScrapeMetadata(\n params: ScrapeLinksParams,\n metrics: ScrapeMetrics,\n tokensPerUrl: number,\n totalBatches: number,\n executionTime: number,\n): ScrapeLinksOutput['metadata'] {\n return {\n total_items: params.urls.length,\n successful: metrics.successful,\n failed: metrics.failed,\n execution_time_ms: executionTime,\n total_credits: metrics.totalCredits,\n };\n}\n\nfunction buildScrapeResponse(\n params: ScrapeLinksParams,\n contents: string[],\n metrics: ScrapeMetrics,\n tokensPerUrl: number,\n totalBatches: number,\n llmErrors: number,\n executionTime: number,\n llmAccounting: { llmAttempted: number; llmSucceeded: boolean },\n): { content: string; structuredContent: ScrapeLinksOutput } {\n // Be explicit about LLM accounting: how many URLs the LLM tried, how many\n // succeeded, and whether the pass produced any value. Without this, the\n // batch header reports `Successful: N + LLM extraction failures: N` and the\n // caller can't tell whether the credits charged for the LLM pass produced\n // anything.\n const llmExtras: Record<string, string | number> = {};\n if (llmAccounting.llmAttempted > 0) {\n const ok = llmAccounting.llmAttempted - llmErrors;\n llmExtras['LLM extraction'] = `${ok}/${llmAccounting.llmAttempted} succeeded`;\n if (!llmAccounting.llmSucceeded) {\n llmExtras['LLM credit'] = '0 charged (no extraction produced)';\n }\n } else if (llmErrors > 0) {\n llmExtras['LLM extraction failures'] = llmErrors;\n }\n\n const batchHeader = formatBatchHeader({\n title: `Scraped Content (${params.urls.length} URLs)`,\n totalItems: params.urls.length,\n successful: metrics.successful,\n failed: metrics.failed,\n tokensPerItem: tokensPerUrl,\n batches: totalBatches,\n extras: {\n 'Credits used': metrics.totalCredits,\n ...llmExtras,\n },\n });\n\n // No cookie-cutter \"Next Steps\" with literal `[...]` placeholders. The\n // server omits the Next Steps block when there are no concrete suggestions\n // to make. See: docs/code-review/context/07-derailment-evidence.md\n // ([FOOTER-BAD]) and mcp-revisions/output-shaping/05.\n const formattedContent = formatSuccess({\n title: 'Scraping Complete',\n summary: batchHeader,\n data: contents.join('\\n\\n---\\n\\n'),\n metadata: {\n 'Execution time': formatDuration(executionTime),\n 'Token budget': TOKEN_BUDGETS.SCRAPER.toLocaleString(),\n },\n });\n\n const metadata = buildScrapeMetadata(params, metrics, tokensPerUrl, totalBatches, executionTime);\n return { content: formattedContent, structuredContent: { content: formattedContent, metadata } };\n}\n\n/**\n * Handle scrape links request\n * NEVER throws - always returns a valid response with content and metadata\n */\nexport async function handleScrapeLinks(\n params: ScrapeLinksParams,\n reporter: ToolReporter = NOOP_REPORTER,\n): Promise<ToolExecutionResult<ScrapeLinksOutput>> {\n const startTime = Date.now();\n\n if (!params.urls || params.urls.length === 0) {\n return createScrapeErrorResponse('NO_URLS', 'No URLs provided', startTime, params.urls?.length || 0);\n }\n\n // Reddit URLs are structurally different (threaded comments, SPA shell,\n // Cloudflare-fronted JSON API). Reject the entire batch so the caller\n // re-routes to get-reddit-post \u2014 silent partial-routing is the\n // [FOOTER-BAD] / [REDUNDANT] anti-pattern from the derailment evidence.\n // See: mcp-revisions/tool-surface/03-reddit-url-routing-in-scrape-links.md\n const redditUrls = params.urls.filter(isRedditUrl);\n if (redditUrls.length > 0) {\n return createScrapeErrorResponse(\n 'UNSUPPORTED_URL_TYPE',\n `scrape-links does not support Reddit URLs. Use get-reddit-post for: ${redditUrls.join(', ')}`,\n startTime,\n params.urls.length,\n false,\n [\n `get-reddit-post(urls=[${redditUrls.map((u) => `\"${u}\"`).join(', ')}]) \u2014 fetch threaded posts and comments directly`,\n 'web-search(queries=[\"...\"], extract=\"...\", scope: \"reddit\") \u2014 find Reddit post permalinks first if these URLs were guesses',\n ],\n );\n }\n\n const { validUrls, invalidUrls } = validateAndPartitionUrls(params.urls);\n await reporter.log('info', `Validated ${validUrls.length} scrapeable URL(s) and ${invalidUrls.length} invalid URL(s)`);\n\n if (validUrls.length === 0) {\n return createScrapeErrorResponse('INVALID_URLS', `All ${params.urls.length} URLs are invalid`, startTime, params.urls.length, false, [\n 'web-search(queries=[\"topic documentation\", \"topic guide\"], extract=\"relevant documentation and guides\") \u2014 search for valid URLs first, then scrape the results',\n 'web-search(queries=[\"topic recommendations\"], extract=\"...\", scope: \"reddit\") \u2014 find Reddit discussion permalinks to feed get-reddit-post instead',\n ]);\n }\n\n const tokensPerUrl = calculateTokenAllocation(validUrls.length, TOKEN_BUDGETS.SCRAPER);\n const totalBatches = Math.ceil(validUrls.length / SCRAPER.BATCH_SIZE);\n\n mcpLog('info', `Starting scrape: ${validUrls.length} URL(s), ${tokensPerUrl} tokens/URL, ${totalBatches} batch(es)`, 'scrape');\n await reporter.progress(15, 100, 'Preparing scraper clients');\n\n let clients: ScrapeClients;\n try {\n clients = initializeScrapeClients();\n } catch (error) {\n const err = classifyError(error);\n return createScrapeErrorResponse('CLIENT_INIT_FAILED', `Failed to initialize scraper: ${err.message}`, startTime, params.urls.length, false, [\n 'web-search(queries=[\"topic key findings\", \"topic summary\", \"topic overview\"], extract=\"key findings and summary\") \u2014 search for information instead of scraping',\n 'web-search(queries=[\"topic discussion\", \"topic recommendations\"], extract=\"...\", scope: \"reddit\") \u2014 get community insights as an alternative',\n ]);\n }\n\n const enhancedInstruction = enhanceExtractionInstruction(params.extract);\n\n await reporter.progress(35, 100, 'Fetching page content');\n const results = await clients.client.scrapeMultiple(validUrls, { timeout: 60 });\n mcpLog('info', `Scraping complete. Processing ${results.length} results...`, 'scrape');\n await reporter.log('info', `Fetched ${results.length} scrape response(s) from the provider`);\n await reporter.progress(60, 100, 'Cleaning and classifying scrape results');\n\n const { successItems, failedContents, metrics } = processScrapeResults(results, invalidUrls);\n\n if (successItems.length > 0) {\n await reporter.progress(80, 100, 'Running LLM extraction over scraped pages');\n }\n const { items: processedItems, llmErrors, llmAttempted } = await processItemsWithLlm(\n successItems, enhancedInstruction, tokensPerUrl, clients.llmProcessor, reporter,\n );\n\n const contents = assembleContentEntries(processedItems, failedContents);\n const executionTime = Date.now() - startTime;\n\n mcpLog('info', `Completed: ${metrics.successful} successful, ${metrics.failed} failed, ${metrics.totalCredits} credits used`, 'scrape');\n await reporter.log(\n 'info',\n `Scrape completed with ${metrics.successful} success(es), ${metrics.failed} failure(s), and ${llmErrors} LLM extraction issue(s)`,\n );\n\n // Credit policy: when LLM was attempted but every URL failed extraction,\n // we still report the failures but make the credit reality explicit in\n // the batch header so callers don't see \"credits used\" without knowing\n // they paid for a no-op LLM pass. See:\n // mcp-revisions/llm-degradation/03-extraction-failure-credit-policy.md\n const llmSucceeded = llmAttempted > 0 && llmErrors < llmAttempted;\n const result = buildScrapeResponse(\n params,\n contents,\n metrics,\n tokensPerUrl,\n totalBatches,\n llmErrors,\n executionTime,\n { llmAttempted, llmSucceeded },\n );\n\n // Contract: every URL failed \u2192 return isError: true so callers that check\n // response.isError can short-circuit. Partial success still resolves\n // through toolSuccess so the agent sees both rows. See\n // docs/code-review/context/02-current-tool-surface.md (E6).\n if (metrics.successful === 0 && metrics.failed > 0) {\n return toolFailure(result.content);\n }\n\n return toolSuccess(result.content, result.structuredContent);\n}\n\nexport function registerScrapeLinksTool(server: MCPServer): void {\n server.tool(\n {\n name: 'scrape-links',\n title: 'Scrape Links',\n description:\n 'Scrape many web pages in parallel and run structured LLM extraction on each (no hard cap). Per-page output: `## Source` (URL + detected page type + date), `## Matches` (verbatim-preserved facts), `## Not found` (explicit gaps the page did not answer \u2014 kills hallucination), `## Follow-up signals` (new terms and referenced-but-unscraped URLs that feed the next research loop), optional `## Contradictions` and `## Truncation` footers. Extraction behavior adapts per page type (docs, github-thread, reddit, marketing, cve, paper, announcement, qa, blog, changelog, release-notes). Use the `extract` field to describe the shape of what you want, separated by `|`.',\n schema: scrapeLinksParamsSchema,\n outputSchema: scrapeLinksOutputSchema,\n annotations: {\n readOnlyHint: true,\n idempotentHint: true,\n destructiveHint: false,\n openWorldHint: true,\n // See contract-fixes/03 \u2014 non-standard precondition hint.\n ...({ experimental: { requires: ['start-research'] } } as Record<string, unknown>),\n },\n _meta: { requires: ['start-research'] },\n },\n async (args, ctx) => {\n if (!getCapabilities().scraping) {\n return toToolResponse(toolFailure(getMissingEnvMessage('scraping')));\n }\n\n const guard = await requireBootstrap(ctx);\n if (guard) {\n return guard;\n }\n\n const reporter = createToolReporter(ctx, 'scrape-links');\n const result = await handleScrapeLinks(args, reporter);\n\n await reporter.progress(100, 100, result.isError ? 'Scrape failed' : 'Scrape complete');\n return toToolResponse(result);\n },\n );\n}\n"],
|
|
5
|
+
"mappings": "AAOA,SAAS,SAAS,aAAa,iBAAiB,4BAA4B;AAC5E;AAAA,EACE;AAAA,EACA;AAAA,OAGK;AACP,SAAS,qBAAqB;AAC9B,SAAS,uBAAuB;AAChC,SAAS,oBAAoB,6BAA6B;AAC1D,SAAS,sBAAsB;AAC/B,SAAS,8BAA8B;AACvC,SAAS,qBAAqB;AAC9B,SAAS,YAAY;AACrB;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OAGK;AACP,SAAS,wBAAwB;AAGjC,MAAM,kBAAkB,IAAI,gBAAgB;AAG5C,SAAS,sBAA8B;AACrC,SAAO,QAAQ;AACjB;AAEA,SAAS,sBAA8B;AACrC,SAAO,QAAQ;AACjB;AAEA,SAAS,6BAA6B,aAAyC;AAC7E,QAAM,OAAO,eAAe;AAC5B,SAAO,GAAG,oBAAoB,CAAC;AAAA;AAAA,EAAO,IAAI;AAAA;AAAA,EAAO,oBAAoB,CAAC;AACxE;AA6BA,SAAS,0BACP,MACA,SACA,WACA,WACA,YAAY,OACZ,cACwC;AACxC,SAAO;AAAA,IACL,GAAG,YAAY;AAAA,MACb;AAAA,MACA;AAAA,MACA;AAAA,MACA,UAAU;AAAA,MACV,UAAU,SAAS,YAAY,CAAC,gCAAgC,IAAI;AAAA,MACpE;AAAA,IACF,CAAC,CAAC;AAAA;AAAA,kBAAuB,eAAe,KAAK,IAAI,IAAI,SAAS,CAAC;AAAA,EACjE;AACF;AAGA,MAAM,cAAc;AAEpB,SAAS,YAAY,KAAsB;AACzC,MAAI;AACF,WAAO,YAAY,KAAK,IAAI,IAAI,GAAG,EAAE,QAAQ;AAAA,EAC/C,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEA,SAAS,yBAAyB,MAAgE;AAChG,QAAM,YAAsB,CAAC;AAC7B,QAAM,cAAwB,CAAC;AAC/B,aAAW,OAAO,MAAM;AACtB,QAAI;AACF,UAAI,IAAI,GAAG;AACX,gBAAU,KAAK,GAAG;AAAA,IACpB,QAAQ;AACN,kBAAY,KAAK,GAAG;AAAA,IACtB;AAAA,EACF;AACA,SAAO,EAAE,WAAW,YAAY;AAClC;AAEA,SAAS,0BAAyC;AAChD,QAAM,SAAS,IAAI,cAAc;AACjC,QAAM,eAAe,mBAAmB;AACxC,SAAO,EAAE,QAAQ,aAAa;AAChC;AAEA,SAAS,qBACP,SACA,aACmB;AACnB,QAAM,eAAkC,CAAC;AACzC,QAAM,iBAA2B,CAAC;AAClC,MAAI,aAAa;AACjB,MAAI,SAAS;AACb,MAAI,eAAe;AAEnB,aAAW,cAAc,aAAa;AACpC;AACA,mBAAe,KAAK,MAAM,UAAU;AAAA;AAAA,0BAA0B;AAAA,EAChE;AAEA,WAAS,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAAK;AACvC,UAAM,SAAS,QAAQ,CAAC;AACxB,QAAI,CAAC,QAAQ;AACX;AACA,qBAAe,KAAK;AAAA;AAAA,0BAAwC;AAC5D;AAAA,IACF;AAEA,WAAO,SAAS,IAAI,IAAI,CAAC,IAAI,QAAQ,MAAM,gBAAgB,OAAO,GAAG,IAAI,QAAQ;AAEjF,QAAI,OAAO,SAAS,OAAO,aAAa,OAAO,OAAO,cAAc,KAAK;AACvE;AACA,YAAM,WAAW,OAAO,OAAO,WAAW,OAAO,WAAW,QAAQ,OAAO,UAAU;AACrF,qBAAe,KAAK,MAAM,OAAO,GAAG;AAAA;AAAA,2BAA2B,QAAQ,EAAE;AACzE,aAAO,WAAW,IAAI,IAAI,CAAC,IAAI,QAAQ,MAAM,aAAa,QAAQ,IAAI,QAAQ;AAC9E;AAAA,IACF;AAEA;AACA,oBAAgB,OAAO;AAKvB,QAAI;AACJ,QAAI;AACF,YAAM,WAAW,uBAAuB,OAAO,SAAS,OAAO,GAAG;AAClE,YAAM,mBAAmB,SAAS,YAAY,SAAS,UAAU,OAAO;AACxE,gBAAU,gBAAgB,eAAe,gBAAgB;AAAA,IAC3D,QAAQ;AACN,gBAAU,OAAO;AAAA,IACnB;AAEA,iBAAa,KAAK,EAAE,KAAK,OAAO,KAAK,SAAS,OAAO,EAAE,CAAC;AAAA,EAC1D;AAEA,SAAO,EAAE,cAAc,gBAAgB,SAAS,EAAE,YAAY,QAAQ,aAAa,EAAE;AACvF;AAEA,eAAe,oBACb,cACA,qBACA,cACA,cACA,UACgF;AAChF,MAAI,YAAY;AAEhB,MAAI,CAAC,gBAAgB,aAAa,WAAW,GAAG;AAC9C,QAAI,CAAC,gBAAgB,aAAa,SAAS,GAAG;AAC5C,aAAO,WAAW,oFAAoF,QAAQ;AAG9G,WAAK,SAAS,IAAI,WAAW,iFAAiF;AAAA,IAChH;AACA,WAAO,EAAE,OAAO,cAAc,WAAW,cAAc,EAAE;AAAA,EAC3D;AAEA,SAAO,QAAQ,wCAAwC,aAAa,MAAM,wBAAwB,YAAY,cAAc,KAAK,QAAQ;AAEzI,QAAM,aAAa,MAAM,KAAK,cAAc,OAAO,SAAS;AAC1D,WAAO,SAAS,kBAAkB,KAAK,GAAG,KAAK,YAAY,eAAe,QAAQ;AAElF,UAAM,YAAY,MAAM;AAAA,MACtB,KAAK;AAAA,MACL,EAAE,SAAS,MAAM,SAAS,qBAAqB,YAAY,cAAc,KAAK,KAAK,IAAI;AAAA,MACvF;AAAA,IACF;AAEA,QAAI,UAAU,WAAW;AACvB,aAAO,SAAS,+BAA+B,KAAK,GAAG,IAAI,QAAQ;AACnE,aAAO,EAAE,GAAG,MAAM,SAAS,UAAU,QAAQ;AAAA,IAC/C;AAEA;AACA,WAAO,WAAW,6BAA6B,KAAK,GAAG,KAAK,UAAU,SAAS,gBAAgB,IAAI,QAAQ;AAC3G,SAAK,SAAS,IAAI,WAAW,8BAA8B,KAAK,GAAG,WAAM,UAAU,SAAS,gBAAgB,EAAE;AAC9G,WAAO;AAAA,EACT,GAAG,YAAY,cAAc;AAE7B,SAAO,EAAE,OAAO,YAAY,WAAW,cAAc,aAAa,OAAO;AAC3E;AAEA,SAAS,uBAAuB,cAAiC,gBAAoC;AACnG,QAAM,WAAW,CAAC,GAAG,cAAc;AACnC,aAAW,QAAQ,cAAc;AAC/B,QAAI,UAAU,KAAK;AACnB,QAAI;AACF,gBAAU,eAAe,OAAO;AAAA,IAClC,QAAQ;AAAA,IAER;AACA,aAAS,KAAK,MAAM,KAAK,GAAG;AAAA;AAAA,EAAO,OAAO,EAAE;AAAA,EAC9C;AACA,SAAO;AACT;AAEA,SAAS,oBACP,QACA,SACA,cACA,cACA,eAC+B;AAC/B,SAAO;AAAA,IACL,aAAa,OAAO,KAAK;AAAA,IACzB,YAAY,QAAQ;AAAA,IACpB,QAAQ,QAAQ;AAAA,IAChB,mBAAmB;AAAA,IACnB,eAAe,QAAQ;AAAA,EACzB;AACF;AAEA,SAAS,oBACP,QACA,UACA,SACA,cACA,cACA,WACA,eACA,eAC2D;AAM3D,QAAM,YAA6C,CAAC;AACpD,MAAI,cAAc,eAAe,GAAG;AAClC,UAAM,KAAK,cAAc,eAAe;AACxC,cAAU,gBAAgB,IAAI,GAAG,EAAE,IAAI,cAAc,YAAY;AACjE,QAAI,CAAC,cAAc,cAAc;AAC/B,gBAAU,YAAY,IAAI;AAAA,IAC5B;AAAA,EACF,WAAW,YAAY,GAAG;AACxB,cAAU,yBAAyB,IAAI;AAAA,EACzC;AAEA,QAAM,cAAc,kBAAkB;AAAA,IACpC,OAAO,oBAAoB,OAAO,KAAK,MAAM;AAAA,IAC7C,YAAY,OAAO,KAAK;AAAA,IACxB,YAAY,QAAQ;AAAA,IACpB,QAAQ,QAAQ;AAAA,IAChB,eAAe;AAAA,IACf,SAAS;AAAA,IACT,QAAQ;AAAA,MACN,gBAAgB,QAAQ;AAAA,MACxB,GAAG;AAAA,IACL;AAAA,EACF,CAAC;AAMD,QAAM,mBAAmB,cAAc;AAAA,IACrC,OAAO;AAAA,IACP,SAAS;AAAA,IACT,MAAM,SAAS,KAAK,aAAa;AAAA,IACjC,UAAU;AAAA,MACR,kBAAkB,eAAe,aAAa;AAAA,MAC9C,gBAAgB,cAAc,QAAQ,eAAe;AAAA,IACvD;AAAA,EACF,CAAC;AAED,QAAM,WAAW,oBAAoB,QAAQ,SAAS,cAAc,cAAc,aAAa;AAC/F,SAAO,EAAE,SAAS,kBAAkB,mBAAmB,EAAE,SAAS,kBAAkB,SAAS,EAAE;AACjG;AAMA,eAAsB,kBACpB,QACA,WAAyB,eACwB;AACjD,QAAM,YAAY,KAAK,IAAI;AAE3B,MAAI,CAAC,OAAO,QAAQ,OAAO,KAAK,WAAW,GAAG;AAC5C,WAAO,0BAA0B,WAAW,oBAAoB,WAAW,OAAO,MAAM,UAAU,CAAC;AAAA,EACrG;AAOA,QAAM,aAAa,OAAO,KAAK,OAAO,WAAW;AACjD,MAAI,WAAW,SAAS,GAAG;AACzB,WAAO;AAAA,MACL;AAAA,MACA,uEAAuE,WAAW,KAAK,IAAI,CAAC;AAAA,MAC5F;AAAA,MACA,OAAO,KAAK;AAAA,MACZ;AAAA,MACA;AAAA,QACE,yBAAyB,WAAW,IAAI,CAAC,MAAM,IAAI,CAAC,GAAG,EAAE,KAAK,IAAI,CAAC;AAAA,QACnE;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAEA,QAAM,EAAE,WAAW,YAAY,IAAI,yBAAyB,OAAO,IAAI;AACvE,QAAM,SAAS,IAAI,QAAQ,aAAa,UAAU,MAAM,0BAA0B,YAAY,MAAM,iBAAiB;AAErH,MAAI,UAAU,WAAW,GAAG;AAC1B,WAAO,0BAA0B,gBAAgB,OAAO,OAAO,KAAK,MAAM,qBAAqB,WAAW,OAAO,KAAK,QAAQ,OAAO;AAAA,MACnI;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAEA,QAAM,eAAe,yBAAyB,UAAU,QAAQ,cAAc,OAAO;AACrF,QAAM,eAAe,KAAK,KAAK,UAAU,SAAS,QAAQ,UAAU;AAEpE,SAAO,QAAQ,oBAAoB,UAAU,MAAM,YAAY,YAAY,gBAAgB,YAAY,cAAc,QAAQ;AAC7H,QAAM,SAAS,SAAS,IAAI,KAAK,2BAA2B;AAE5D,MAAI;AACJ,MAAI;AACF,cAAU,wBAAwB;AAAA,EACpC,SAAS,OAAO;AACd,UAAM,MAAM,cAAc,KAAK;AAC/B,WAAO,0BAA0B,sBAAsB,iCAAiC,IAAI,OAAO,IAAI,WAAW,OAAO,KAAK,QAAQ,OAAO;AAAA,MAC3I;AAAA,MACA;AAAA,IACF,CAAC;AAAA,EACH;AAEA,QAAM,sBAAsB,6BAA6B,OAAO,OAAO;AAEvE,QAAM,SAAS,SAAS,IAAI,KAAK,uBAAuB;AACxD,QAAM,UAAU,MAAM,QAAQ,OAAO,eAAe,WAAW,EAAE,SAAS,GAAG,CAAC;AAC9E,SAAO,QAAQ,iCAAiC,QAAQ,MAAM,eAAe,QAAQ;AACrF,QAAM,SAAS,IAAI,QAAQ,WAAW,QAAQ,MAAM,uCAAuC;AAC3F,QAAM,SAAS,SAAS,IAAI,KAAK,yCAAyC;AAE1E,QAAM,EAAE,cAAc,gBAAgB,QAAQ,IAAI,qBAAqB,SAAS,WAAW;AAE3F,MAAI,aAAa,SAAS,GAAG;AAC3B,UAAM,SAAS,SAAS,IAAI,KAAK,2CAA2C;AAAA,EAC9E;AACA,QAAM,EAAE,OAAO,gBAAgB,WAAW,aAAa,IAAI,MAAM;AAAA,IAC/D;AAAA,IAAc;AAAA,IAAqB;AAAA,IAAc,QAAQ;AAAA,IAAc;AAAA,EACzE;AAEA,QAAM,WAAW,uBAAuB,gBAAgB,cAAc;AACtE,QAAM,gBAAgB,KAAK,IAAI,IAAI;AAEnC,SAAO,QAAQ,cAAc,QAAQ,UAAU,gBAAgB,QAAQ,MAAM,YAAY,QAAQ,YAAY,iBAAiB,QAAQ;AACtI,QAAM,SAAS;AAAA,IACb;AAAA,IACA,yBAAyB,QAAQ,UAAU,iBAAiB,QAAQ,MAAM,oBAAoB,SAAS;AAAA,EACzG;AAOA,QAAM,eAAe,eAAe,KAAK,YAAY;AACrD,QAAM,SAAS;AAAA,IACb;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,EAAE,cAAc,aAAa;AAAA,EAC/B;AAMA,MAAI,QAAQ,eAAe,KAAK,QAAQ,SAAS,GAAG;AAClD,WAAO,YAAY,OAAO,OAAO;AAAA,EACnC;AAEA,SAAO,YAAY,OAAO,SAAS,OAAO,iBAAiB;AAC7D;AAEO,SAAS,wBAAwB,QAAyB;AAC/D,SAAO;AAAA,IACL;AAAA,MACE,MAAM;AAAA,MACN,OAAO;AAAA,MACP,aACE;AAAA,MACF,QAAQ;AAAA,MACR,cAAc;AAAA,MACd,aAAa;AAAA,QACX,cAAc;AAAA,QACd,gBAAgB;AAAA,QAChB,iBAAiB;AAAA,QACjB,eAAe;AAAA;AAAA,QAEf,GAAI,EAAE,cAAc,EAAE,UAAU,CAAC,gBAAgB,EAAE,EAAE;AAAA,MACvD;AAAA,MACA,OAAO,EAAE,UAAU,CAAC,gBAAgB,EAAE;AAAA,IACxC;AAAA,IACA,OAAO,MAAM,QAAQ;AACnB,UAAI,CAAC,gBAAgB,EAAE,UAAU;AAC/B,eAAO,eAAe,YAAY,qBAAqB,UAAU,CAAC,CAAC;AAAA,MACrE;AAEA,YAAM,QAAQ,MAAM,iBAAiB,GAAG;AACxC,UAAI,OAAO;AACT,eAAO;AAAA,MACT;AAEA,YAAM,WAAW,mBAAmB,KAAK,cAAc;AACvD,YAAM,SAAS,MAAM,kBAAkB,MAAM,QAAQ;AAErD,YAAM,SAAS,SAAS,KAAK,KAAK,OAAO,UAAU,kBAAkB,iBAAiB;AACtF,aAAO,eAAe,MAAM;AAAA,IAC9B;AAAA,EACF;AACF;",
|
|
6
|
+
"names": []
|
|
7
|
+
}
|