mcp-researchpowerpack 7.0.11 → 7.0.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +4662 -21
- package/dist/index.js.map +4 -4
- package/dist/mcp-use.json +2 -2
- package/dist/src/clients/jina.js +202 -16
- package/dist/src/clients/jina.js.map +3 -3
- package/dist/src/clients/kernel.js +254 -7
- package/dist/src/clients/kernel.js.map +4 -4
- package/dist/src/clients/reddit.js +326 -23
- package/dist/src/clients/reddit.js.map +4 -4
- package/dist/src/clients/scraper.js +345 -22
- package/dist/src/clients/scraper.js.map +4 -4
- package/dist/src/clients/search.js +316 -20
- package/dist/src/clients/search.js.map +4 -4
- package/dist/src/config/index.js +39 -10
- package/dist/src/config/index.js.map +3 -3
- package/dist/src/effect/errors.js +130 -5
- package/dist/src/effect/errors.js.map +3 -3
- package/dist/src/effect/runtime.js +1893 -4
- package/dist/src/effect/runtime.js.map +4 -4
- package/dist/src/effect/services.js +2124 -22
- package/dist/src/effect/services.js.map +4 -4
- package/dist/src/schemas/scrape-links.js +6 -5
- package/dist/src/schemas/scrape-links.js.map +1 -1
- package/dist/src/schemas/start-research.js +2 -1
- package/dist/src/schemas/start-research.js.map +1 -1
- package/dist/src/schemas/web-search.js +9 -8
- package/dist/src/schemas/web-search.js.map +1 -1
- package/dist/src/services/llm-processor.js +406 -25
- package/dist/src/services/llm-processor.js.map +4 -4
- package/dist/src/services/markdown-cleaner.js +6 -5
- package/dist/src/services/markdown-cleaner.js.map +1 -1
- package/dist/src/tools/mcp-helpers.js +2 -1
- package/dist/src/tools/mcp-helpers.js.map +1 -1
- package/dist/src/tools/registry.js +4629 -3
- package/dist/src/tools/registry.js.map +4 -4
- package/dist/src/tools/scrape.js +2610 -80
- package/dist/src/tools/scrape.js.map +4 -4
- package/dist/src/tools/search.js +2388 -59
- package/dist/src/tools/search.js.map +4 -4
- package/dist/src/tools/start-research.js +2030 -23
- package/dist/src/tools/start-research.js.map +4 -4
- package/dist/src/tools/utils.js +98 -7
- package/dist/src/tools/utils.js.map +3 -3
- package/dist/src/utils/concurrency.js +1 -0
- package/dist/src/utils/concurrency.js.map +1 -1
- package/dist/src/utils/content-extractor.js +27 -2
- package/dist/src/utils/content-extractor.js.map +3 -3
- package/dist/src/utils/content-quality.js +4 -3
- package/dist/src/utils/content-quality.js.map +1 -1
- package/dist/src/utils/errors.js +26 -3
- package/dist/src/utils/errors.js.map +3 -3
- package/dist/src/utils/logger.js +1 -0
- package/dist/src/utils/logger.js.map +1 -1
- package/dist/src/utils/markdown-formatter.js +1 -0
- package/dist/src/utils/markdown-formatter.js.map +1 -1
- package/dist/src/utils/query-relax.js +9 -8
- package/dist/src/utils/query-relax.js.map +1 -1
- package/dist/src/utils/response.js +3 -2
- package/dist/src/utils/response.js.map +1 -1
- package/dist/src/utils/retry.js +5 -4
- package/dist/src/utils/retry.js.map +1 -1
- package/dist/src/utils/sanitize.js +4 -3
- package/dist/src/utils/sanitize.js.map +1 -1
- package/dist/src/utils/source-type.js +4 -3
- package/dist/src/utils/source-type.js.map +1 -1
- package/dist/src/utils/url-aggregator.js +112 -11
- package/dist/src/utils/url-aggregator.js.map +3 -3
- package/dist/src/version.js +7 -6
- package/dist/src/version.js.map +1 -1
- package/package.json +3 -3
|
@@ -1,31 +1,32 @@
|
|
|
1
|
+
// src/schemas/web-search.ts
|
|
1
2
|
import { z } from "zod";
|
|
2
|
-
|
|
3
|
+
var QUERY_REWRITE_PAIR_EXAMPLES = [
|
|
3
4
|
'Bad: `<feature> support` \u2192 Better: `site:<official-docs-domain> "<feature>" "<platform-or-version>"`',
|
|
4
5
|
'Bad: `<product> pricing` \u2192 Better: `site:<vendor-domain> "<product>" pricing "enterprise" OR "free tier"`',
|
|
5
6
|
'Bad: `<library> bug fix` \u2192 Better: `"<exact error text>" "<library-or-package>" "<version>" site:github.com`',
|
|
6
7
|
'Bad: `<tool> reviews` \u2192 Better: `site:reddit.com/r/<community>/comments "<tool>" "migration" OR "regression"`'
|
|
7
8
|
];
|
|
8
|
-
|
|
9
|
+
var QUERY_REWRITE_PAIR_GUIDANCE = [
|
|
9
10
|
"Write Google retrieval probes, not topic labels.",
|
|
10
11
|
"For each broad idea, rewrite it into a query that names the evidence source class, discriminating anchor terms, and one useful operator when possible.",
|
|
11
12
|
"Use rewrite-pair thinking before searching:",
|
|
12
13
|
...QUERY_REWRITE_PAIR_EXAMPLES,
|
|
13
14
|
"Do not repeat the same noun phrase with adjectives changed; fan out by source type and evidence need."
|
|
14
15
|
];
|
|
15
|
-
|
|
16
|
-
|
|
16
|
+
var QUERY_REWRITE_PAIR_GUIDANCE_TEXT = QUERY_REWRITE_PAIR_GUIDANCE.join(" ");
|
|
17
|
+
var keywordSchema = z.string().min(1, { message: "search: Keyword cannot be empty" }).describe(
|
|
17
18
|
`A single search keyword/query. Each item runs as a separate parallel search. ${QUERY_REWRITE_PAIR_GUIDANCE_TEXT}`
|
|
18
19
|
);
|
|
19
|
-
|
|
20
|
+
var keywordsSchema = z.array(keywordSchema).min(1, { message: "search: At least 1 keyword required" }).max(50, { message: "search: At most 50 keywords allowed per call" }).describe(
|
|
20
21
|
`Search keywords to run in parallel. Serper is primary when configured; Jina Search is fallback when Serper is missing, fails, or yields empty query results. ${QUERY_REWRITE_PAIR_GUIDANCE_TEXT} Think of keywords as retrieval probes, not topic labels. Pack distinct facets in one call: official docs, implementation, failures, comparisons, sentiment, changelog, CVE, pricing, or other source classes.`
|
|
21
22
|
);
|
|
22
|
-
|
|
23
|
+
var rawWebSearchParamsSchema = z.object({
|
|
23
24
|
keywords: keywordsSchema,
|
|
24
25
|
extract: z.never().optional(),
|
|
25
26
|
scope: z.never().optional(),
|
|
26
27
|
verbose: z.never().optional()
|
|
27
28
|
}).strict();
|
|
28
|
-
|
|
29
|
+
var smartWebSearchParamsSchema = z.object({
|
|
29
30
|
keywords: keywordsSchema,
|
|
30
31
|
extract: z.string().min(1, { message: "smart-web-search: extract cannot be empty" }).describe(
|
|
31
32
|
'Semantic instruction for the relevance classifier \u2014 what "relevant" means for THIS goal. This is the post-sort target, so name the evidence you need and the source-of-truth expectation: e.g. official docs/release notes for specs, issue/PR/error text for bugs, Reddit/HN/blogs for lived experience, vendor pricing pages for pricing, CVE databases for security. Drives tiering (HIGHLY_RELEVANT / MAYBE_RELEVANT / OTHER), synthesis, gap analysis, and refine-query suggestions. Be specific: "OAuth 2.1 support in TypeScript MCP frameworks \u2014 runnable code, not marketing", not "MCP OAuth".'
|
|
@@ -37,7 +38,7 @@ const smartWebSearchParamsSchema = z.object({
|
|
|
37
38
|
"Include per-row scoring/coverage metadata, the trailing Signals block, and CONSENSUS labels even when they carry little signal. Default false."
|
|
38
39
|
)
|
|
39
40
|
}).strict();
|
|
40
|
-
|
|
41
|
+
var webSearchParamsSchema = smartWebSearchParamsSchema;
|
|
41
42
|
export {
|
|
42
43
|
QUERY_REWRITE_PAIR_EXAMPLES,
|
|
43
44
|
QUERY_REWRITE_PAIR_GUIDANCE,
|
|
@@ -2,6 +2,6 @@
|
|
|
2
2
|
"version": 3,
|
|
3
3
|
"sources": ["../../../src/schemas/web-search.ts"],
|
|
4
4
|
"sourcesContent": ["import { z } from 'zod';\n\nexport const QUERY_REWRITE_PAIR_EXAMPLES = [\n 'Bad: `<feature> support` \u2192 Better: `site:<official-docs-domain> \"<feature>\" \"<platform-or-version>\"`',\n 'Bad: `<product> pricing` \u2192 Better: `site:<vendor-domain> \"<product>\" pricing \"enterprise\" OR \"free tier\"`',\n 'Bad: `<library> bug fix` \u2192 Better: `\"<exact error text>\" \"<library-or-package>\" \"<version>\" site:github.com`',\n 'Bad: `<tool> reviews` \u2192 Better: `site:reddit.com/r/<community>/comments \"<tool>\" \"migration\" OR \"regression\"`',\n] as const;\n\nexport const QUERY_REWRITE_PAIR_GUIDANCE = [\n 'Write Google retrieval probes, not topic labels.',\n 'For each broad idea, rewrite it into a query that names the evidence source class, discriminating anchor terms, and one useful operator when possible.',\n 'Use rewrite-pair thinking before searching:',\n ...QUERY_REWRITE_PAIR_EXAMPLES,\n 'Do not repeat the same noun phrase with adjectives changed; fan out by source type and evidence need.',\n] as const;\n\nexport const QUERY_REWRITE_PAIR_GUIDANCE_TEXT = QUERY_REWRITE_PAIR_GUIDANCE.join(' ');\n\nconst keywordSchema = z\n .string()\n .min(1, { message: 'search: Keyword cannot be empty' })\n .describe(\n `A single search keyword/query. Each item runs as a separate parallel search. ${QUERY_REWRITE_PAIR_GUIDANCE_TEXT}`,\n );\n\nconst keywordsSchema = z\n .array(keywordSchema)\n .min(1, { message: 'search: At least 1 keyword required' })\n .max(50, { message: 'search: At most 50 keywords allowed per call' })\n .describe(\n `Search keywords to run in parallel. Serper is primary when configured; Jina Search is fallback when Serper is missing, fails, or yields empty query results. ${QUERY_REWRITE_PAIR_GUIDANCE_TEXT} Think of keywords as retrieval probes, not topic labels. Pack distinct facets in one call: official docs, implementation, failures, comparisons, sentiment, changelog, CVE, pricing, or other source classes.`,\n );\n\nexport const rawWebSearchParamsSchema = z.object({\n keywords: keywordsSchema,\n extract: z.never().optional(),\n scope: z.never().optional(),\n verbose: z.never().optional(),\n}).strict();\n\nexport const smartWebSearchParamsSchema = z.object({\n keywords: keywordsSchema,\n extract: z\n .string()\n .min(1, { message: 'smart-web-search: extract cannot be empty' })\n .describe(\n 'Semantic instruction for the relevance classifier \u2014 what \"relevant\" means for THIS goal. This is the post-sort target, so name the evidence you need and the source-of-truth expectation: e.g. official docs/release notes for specs, issue/PR/error text for bugs, Reddit/HN/blogs for lived experience, vendor pricing pages for pricing, CVE databases for security. Drives tiering (HIGHLY_RELEVANT / MAYBE_RELEVANT / OTHER), synthesis, gap analysis, and refine-query suggestions. Be specific: \"OAuth 2.1 support in TypeScript MCP frameworks \u2014 runnable code, not marketing\", not \"MCP OAuth\".',\n ),\n scope: z\n .enum(['web', 'reddit', 'both'])\n .default('web')\n .describe(\n 'Search scope. \"web\" (default) = open web, no augmentation. \"reddit\" = server appends `site:reddit.com` to every keyword and filters results to post permalinks (`/r/.+/comments/[a-z0-9]+/`); subreddit homepages are dropped. \"both\" = runs every keyword twice (open web + reddit-scoped), merges the result set, and tags each row with its source. Use \"reddit\" for sentiment/migration/lived-experience research; use \"both\" when opinion-heavy AND official sources also matter.',\n ),\n verbose: z\n .boolean()\n .default(false)\n .describe(\n 'Include per-row scoring/coverage metadata, the trailing Signals block, and CONSENSUS labels even when they carry little signal. Default false.',\n ),\n}).strict();\n\nexport type RawWebSearchParams = z.infer<typeof rawWebSearchParamsSchema>;\nexport type SmartWebSearchParams = z.infer<typeof smartWebSearchParamsSchema>;\n\n// Internal alias retained for shared tests/helpers. Public tools register the\n// raw/smart schemas above, not this alias.\nexport const webSearchParamsSchema = smartWebSearchParamsSchema;\nexport type WebSearchParams = SmartWebSearchParams;\n\n// Search tools are markdown-only at the MCP boundary.\nexport type WebSearchOutput = Record<string, never>;\n"],
|
|
5
|
-
"mappings": "AAAA,SAAS,SAAS;AAEX,
|
|
5
|
+
"mappings": ";AAAA,SAAS,SAAS;AAEX,IAAM,8BAA8B;AAAA,EACzC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEO,IAAM,8BAA8B;AAAA,EACzC;AAAA,EACA;AAAA,EACA;AAAA,EACA,GAAG;AAAA,EACH;AACF;AAEO,IAAM,mCAAmC,4BAA4B,KAAK,GAAG;AAEpF,IAAM,gBAAgB,EACnB,OAAO,EACP,IAAI,GAAG,EAAE,SAAS,kCAAkC,CAAC,EACrD;AAAA,EACC,gFAAgF,gCAAgC;AAClH;AAEF,IAAM,iBAAiB,EACpB,MAAM,aAAa,EACnB,IAAI,GAAG,EAAE,SAAS,sCAAsC,CAAC,EACzD,IAAI,IAAI,EAAE,SAAS,+CAA+C,CAAC,EACnE;AAAA,EACC,gKAAgK,gCAAgC;AAClM;AAEK,IAAM,2BAA2B,EAAE,OAAO;AAAA,EAC/C,UAAU;AAAA,EACV,SAAS,EAAE,MAAM,EAAE,SAAS;AAAA,EAC5B,OAAO,EAAE,MAAM,EAAE,SAAS;AAAA,EAC1B,SAAS,EAAE,MAAM,EAAE,SAAS;AAC9B,CAAC,EAAE,OAAO;AAEH,IAAM,6BAA6B,EAAE,OAAO;AAAA,EACjD,UAAU;AAAA,EACV,SAAS,EACN,OAAO,EACP,IAAI,GAAG,EAAE,SAAS,4CAA4C,CAAC,EAC/D;AAAA,IACC;AAAA,EACF;AAAA,EACF,OAAO,EACJ,KAAK,CAAC,OAAO,UAAU,MAAM,CAAC,EAC9B,QAAQ,KAAK,EACb;AAAA,IACC;AAAA,EACF;AAAA,EACF,SAAS,EACN,QAAQ,EACR,QAAQ,KAAK,EACb;AAAA,IACC;AAAA,EACF;AACJ,CAAC,EAAE,OAAO;AAOH,IAAM,wBAAwB;",
|
|
6
6
|
"names": []
|
|
7
7
|
}
|
|
@@ -1,20 +1,401 @@
|
|
|
1
|
+
// src/services/llm-processor.ts
|
|
1
2
|
import OpenAI from "openai";
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
import {
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
} from "
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
3
|
+
|
|
4
|
+
// src/config/index.ts
|
|
5
|
+
import { Logger } from "mcp-use";
|
|
6
|
+
|
|
7
|
+
// src/version.ts
|
|
8
|
+
import { createRequire } from "module";
|
|
9
|
+
import { fileURLToPath } from "url";
|
|
10
|
+
import { dirname, join } from "path";
|
|
11
|
+
var DEFAULT_PACKAGE_INFO = {
|
|
12
|
+
version: "3.9.5",
|
|
13
|
+
name: "mcp-researchpowerpack-http",
|
|
14
|
+
description: "Research Powerpack MCP Server"
|
|
15
|
+
};
|
|
16
|
+
var packageJson = { ...DEFAULT_PACKAGE_INFO };
|
|
17
|
+
try {
|
|
18
|
+
if (typeof import.meta.url === "string" && import.meta.url.startsWith("file:")) {
|
|
19
|
+
const _require = createRequire(import.meta.url);
|
|
20
|
+
const _dirname = dirname(fileURLToPath(import.meta.url));
|
|
21
|
+
try {
|
|
22
|
+
packageJson = _require(join(_dirname, "..", "package.json"));
|
|
23
|
+
} catch {
|
|
24
|
+
packageJson = _require(join(_dirname, "..", "..", "package.json"));
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
} catch {
|
|
28
|
+
}
|
|
29
|
+
var VERSION = packageJson.version;
|
|
30
|
+
var PACKAGE_NAME = packageJson.name;
|
|
31
|
+
var PACKAGE_DESCRIPTION = packageJson.description;
|
|
32
|
+
var USER_AGENT_VERSION = `${PACKAGE_NAME}/${VERSION}`;
|
|
33
|
+
|
|
34
|
+
// src/config/index.ts
|
|
35
|
+
function safeParseInt(value, defaultVal, min, max) {
|
|
36
|
+
const logger = Logger.get("config");
|
|
37
|
+
if (!value) {
|
|
38
|
+
return defaultVal;
|
|
39
|
+
}
|
|
40
|
+
const parsed = parseInt(value, 10);
|
|
41
|
+
if (isNaN(parsed)) {
|
|
42
|
+
logger.warn(`Invalid number "${value}", using default ${defaultVal}`);
|
|
43
|
+
return defaultVal;
|
|
44
|
+
}
|
|
45
|
+
if (parsed < min) {
|
|
46
|
+
logger.warn(`Value ${parsed} below minimum ${min}, clamping to ${min}`);
|
|
47
|
+
return min;
|
|
48
|
+
}
|
|
49
|
+
if (parsed > max) {
|
|
50
|
+
logger.warn(`Value ${parsed} above maximum ${max}, clamping to ${max}`);
|
|
51
|
+
return max;
|
|
52
|
+
}
|
|
53
|
+
return parsed;
|
|
54
|
+
}
|
|
55
|
+
var cachedEnv = null;
|
|
56
|
+
function parseEnv() {
|
|
57
|
+
if (cachedEnv) return cachedEnv;
|
|
58
|
+
cachedEnv = {
|
|
59
|
+
SCRAPER_API_KEY: process.env.SCRAPEDO_API_KEY || "",
|
|
60
|
+
SEARCH_API_KEY: process.env.SERPER_API_KEY || void 0,
|
|
61
|
+
REDDIT_CLIENT_ID: process.env.REDDIT_CLIENT_ID || void 0,
|
|
62
|
+
REDDIT_CLIENT_SECRET: process.env.REDDIT_CLIENT_SECRET || void 0,
|
|
63
|
+
JINA_API_KEY: process.env.JINA_API_KEY || void 0,
|
|
64
|
+
KERNEL_API_KEY: process.env.KERNEL_API_KEY || void 0,
|
|
65
|
+
KERNEL_PROJECT: process.env.KERNEL_PROJECT || void 0
|
|
66
|
+
};
|
|
67
|
+
return cachedEnv;
|
|
68
|
+
}
|
|
69
|
+
function getCapabilities() {
|
|
70
|
+
const env = parseEnv();
|
|
71
|
+
return {
|
|
72
|
+
reddit: !!(env.REDDIT_CLIENT_ID && env.REDDIT_CLIENT_SECRET),
|
|
73
|
+
search: !!(env.SEARCH_API_KEY || env.JINA_API_KEY),
|
|
74
|
+
serperSearch: !!env.SEARCH_API_KEY,
|
|
75
|
+
jina: !!env.JINA_API_KEY,
|
|
76
|
+
scraping: !!env.SCRAPER_API_KEY,
|
|
77
|
+
kernel: !!env.KERNEL_API_KEY,
|
|
78
|
+
llmExtraction: getLLMConfigStatus().configured
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
var CONCURRENCY = {
|
|
82
|
+
SEARCH: safeParseInt(process.env.CONCURRENCY_SEARCH, 50, 1, 200),
|
|
83
|
+
SCRAPER: safeParseInt(process.env.CONCURRENCY_SCRAPER, 50, 1, 200),
|
|
84
|
+
JINA_READER: safeParseInt(process.env.CONCURRENCY_JINA_READER, 50, 1, 200),
|
|
85
|
+
REDDIT: safeParseInt(process.env.CONCURRENCY_REDDIT, 50, 1, 200),
|
|
86
|
+
LLM_EXTRACTION: safeParseInt(process.env.LLM_CONCURRENCY, 50, 1, 200),
|
|
87
|
+
KERNEL: safeParseInt(process.env.CONCURRENCY_KERNEL, 3, 1, 20)
|
|
88
|
+
};
|
|
89
|
+
var cachedLlmConfigStatus = null;
|
|
90
|
+
function getLLMConfigStatus() {
|
|
91
|
+
if (cachedLlmConfigStatus) return cachedLlmConfigStatus;
|
|
92
|
+
const apiKeyPresent = !!process.env.LLM_API_KEY?.trim();
|
|
93
|
+
const baseUrlPresent = !!process.env.LLM_BASE_URL?.trim();
|
|
94
|
+
const modelPresent = !!process.env.LLM_MODEL?.trim();
|
|
95
|
+
const missingVars = [];
|
|
96
|
+
if (!apiKeyPresent) missingVars.push("LLM_API_KEY");
|
|
97
|
+
if (!baseUrlPresent) missingVars.push("LLM_BASE_URL");
|
|
98
|
+
if (!modelPresent) missingVars.push("LLM_MODEL");
|
|
99
|
+
const configured = missingVars.length === 0;
|
|
100
|
+
cachedLlmConfigStatus = {
|
|
101
|
+
configured,
|
|
102
|
+
apiKeyPresent,
|
|
103
|
+
baseUrlPresent,
|
|
104
|
+
modelPresent,
|
|
105
|
+
missingVars,
|
|
106
|
+
error: configured ? null : `LLM disabled: missing ${missingVars.join(", ")}`
|
|
107
|
+
};
|
|
108
|
+
return cachedLlmConfigStatus;
|
|
109
|
+
}
|
|
110
|
+
var cachedLlmExtraction = null;
|
|
111
|
+
function getLlmExtraction() {
|
|
112
|
+
if (cachedLlmExtraction) return cachedLlmExtraction;
|
|
113
|
+
const apiKey = process.env.LLM_API_KEY?.trim() || "";
|
|
114
|
+
const baseUrl = process.env.LLM_BASE_URL?.trim();
|
|
115
|
+
const model = process.env.LLM_MODEL?.trim();
|
|
116
|
+
const fallbackModel = process.env.LLM_FALLBACK_MODEL?.trim() || "";
|
|
117
|
+
if (apiKey && !baseUrl) {
|
|
118
|
+
throw new Error(
|
|
119
|
+
"LLM_BASE_URL is required when LLM_API_KEY is set. Set LLM_BASE_URL to your OpenAI-compatible endpoint."
|
|
120
|
+
);
|
|
121
|
+
}
|
|
122
|
+
if (apiKey && !model) {
|
|
123
|
+
throw new Error(
|
|
124
|
+
"LLM_MODEL is required when LLM_API_KEY is set."
|
|
125
|
+
);
|
|
126
|
+
}
|
|
127
|
+
cachedLlmExtraction = {
|
|
128
|
+
API_KEY: apiKey,
|
|
129
|
+
BASE_URL: baseUrl || "",
|
|
130
|
+
MODEL: model || "",
|
|
131
|
+
FALLBACK_MODEL: fallbackModel
|
|
132
|
+
};
|
|
133
|
+
return cachedLlmExtraction;
|
|
134
|
+
}
|
|
135
|
+
var LLM_EXTRACTION = new Proxy({}, {
|
|
136
|
+
get(_target, prop) {
|
|
137
|
+
return getLlmExtraction()[prop];
|
|
138
|
+
}
|
|
139
|
+
});
|
|
140
|
+
|
|
141
|
+
// src/schemas/web-search.ts
|
|
142
|
+
import { z } from "zod";
|
|
143
|
+
var QUERY_REWRITE_PAIR_EXAMPLES = [
|
|
144
|
+
'Bad: `<feature> support` \u2192 Better: `site:<official-docs-domain> "<feature>" "<platform-or-version>"`',
|
|
145
|
+
'Bad: `<product> pricing` \u2192 Better: `site:<vendor-domain> "<product>" pricing "enterprise" OR "free tier"`',
|
|
146
|
+
'Bad: `<library> bug fix` \u2192 Better: `"<exact error text>" "<library-or-package>" "<version>" site:github.com`',
|
|
147
|
+
'Bad: `<tool> reviews` \u2192 Better: `site:reddit.com/r/<community>/comments "<tool>" "migration" OR "regression"`'
|
|
148
|
+
];
|
|
149
|
+
var QUERY_REWRITE_PAIR_GUIDANCE = [
|
|
150
|
+
"Write Google retrieval probes, not topic labels.",
|
|
151
|
+
"For each broad idea, rewrite it into a query that names the evidence source class, discriminating anchor terms, and one useful operator when possible.",
|
|
152
|
+
"Use rewrite-pair thinking before searching:",
|
|
153
|
+
...QUERY_REWRITE_PAIR_EXAMPLES,
|
|
154
|
+
"Do not repeat the same noun phrase with adjectives changed; fan out by source type and evidence need."
|
|
155
|
+
];
|
|
156
|
+
var QUERY_REWRITE_PAIR_GUIDANCE_TEXT = QUERY_REWRITE_PAIR_GUIDANCE.join(" ");
|
|
157
|
+
var keywordSchema = z.string().min(1, { message: "search: Keyword cannot be empty" }).describe(
|
|
158
|
+
`A single search keyword/query. Each item runs as a separate parallel search. ${QUERY_REWRITE_PAIR_GUIDANCE_TEXT}`
|
|
159
|
+
);
|
|
160
|
+
var keywordsSchema = z.array(keywordSchema).min(1, { message: "search: At least 1 keyword required" }).max(50, { message: "search: At most 50 keywords allowed per call" }).describe(
|
|
161
|
+
`Search keywords to run in parallel. Serper is primary when configured; Jina Search is fallback when Serper is missing, fails, or yields empty query results. ${QUERY_REWRITE_PAIR_GUIDANCE_TEXT} Think of keywords as retrieval probes, not topic labels. Pack distinct facets in one call: official docs, implementation, failures, comparisons, sentiment, changelog, CVE, pricing, or other source classes.`
|
|
162
|
+
);
|
|
163
|
+
var rawWebSearchParamsSchema = z.object({
|
|
164
|
+
keywords: keywordsSchema,
|
|
165
|
+
extract: z.never().optional(),
|
|
166
|
+
scope: z.never().optional(),
|
|
167
|
+
verbose: z.never().optional()
|
|
168
|
+
}).strict();
|
|
169
|
+
var smartWebSearchParamsSchema = z.object({
|
|
170
|
+
keywords: keywordsSchema,
|
|
171
|
+
extract: z.string().min(1, { message: "smart-web-search: extract cannot be empty" }).describe(
|
|
172
|
+
'Semantic instruction for the relevance classifier \u2014 what "relevant" means for THIS goal. This is the post-sort target, so name the evidence you need and the source-of-truth expectation: e.g. official docs/release notes for specs, issue/PR/error text for bugs, Reddit/HN/blogs for lived experience, vendor pricing pages for pricing, CVE databases for security. Drives tiering (HIGHLY_RELEVANT / MAYBE_RELEVANT / OTHER), synthesis, gap analysis, and refine-query suggestions. Be specific: "OAuth 2.1 support in TypeScript MCP frameworks \u2014 runnable code, not marketing", not "MCP OAuth".'
|
|
173
|
+
),
|
|
174
|
+
scope: z.enum(["web", "reddit", "both"]).default("web").describe(
|
|
175
|
+
'Search scope. "web" (default) = open web, no augmentation. "reddit" = server appends `site:reddit.com` to every keyword and filters results to post permalinks (`/r/.+/comments/[a-z0-9]+/`); subreddit homepages are dropped. "both" = runs every keyword twice (open web + reddit-scoped), merges the result set, and tags each row with its source. Use "reddit" for sentiment/migration/lived-experience research; use "both" when opinion-heavy AND official sources also matter.'
|
|
176
|
+
),
|
|
177
|
+
verbose: z.boolean().default(false).describe(
|
|
178
|
+
"Include per-row scoring/coverage metadata, the trailing Signals block, and CONSENSUS labels even when they carry little signal. Default false."
|
|
179
|
+
)
|
|
180
|
+
}).strict();
|
|
181
|
+
|
|
182
|
+
// src/utils/logger.ts
|
|
183
|
+
import { Logger as Logger2 } from "mcp-use";
|
|
184
|
+
function getLogger(name) {
|
|
185
|
+
return Logger2.get(name);
|
|
186
|
+
}
|
|
187
|
+
function mcpLog(level, message, loggerName) {
|
|
188
|
+
const logger = getLogger(loggerName ?? "research-powerpack");
|
|
189
|
+
switch (level) {
|
|
190
|
+
case "debug":
|
|
191
|
+
logger.debug(message);
|
|
192
|
+
break;
|
|
193
|
+
case "info":
|
|
194
|
+
logger.info(message);
|
|
195
|
+
break;
|
|
196
|
+
case "warning":
|
|
197
|
+
logger.warn(message);
|
|
198
|
+
break;
|
|
199
|
+
case "error":
|
|
200
|
+
logger.error(message);
|
|
201
|
+
break;
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
// src/utils/errors.ts
|
|
206
|
+
var ErrorCode = {
|
|
207
|
+
// Retryable errors
|
|
208
|
+
RATE_LIMITED: "RATE_LIMITED",
|
|
209
|
+
TIMEOUT: "TIMEOUT",
|
|
210
|
+
NETWORK_ERROR: "NETWORK_ERROR",
|
|
211
|
+
SERVICE_UNAVAILABLE: "SERVICE_UNAVAILABLE",
|
|
212
|
+
// Non-retryable errors
|
|
213
|
+
AUTH_ERROR: "AUTH_ERROR",
|
|
214
|
+
INVALID_INPUT: "INVALID_INPUT",
|
|
215
|
+
NOT_FOUND: "NOT_FOUND",
|
|
216
|
+
QUOTA_EXCEEDED: "QUOTA_EXCEEDED",
|
|
217
|
+
UNSUPPORTED_BINARY_CONTENT: "UNSUPPORTED_BINARY_CONTENT",
|
|
218
|
+
// Internal errors
|
|
219
|
+
INTERNAL_ERROR: "INTERNAL_ERROR",
|
|
220
|
+
PARSE_ERROR: "PARSE_ERROR",
|
|
221
|
+
UNKNOWN_ERROR: "UNKNOWN_ERROR"
|
|
222
|
+
};
|
|
223
|
+
var DEFAULT_RETRY_OPTIONS = {
|
|
224
|
+
maxRetries: 3,
|
|
225
|
+
baseDelayMs: 1e3,
|
|
226
|
+
maxDelayMs: 3e4,
|
|
227
|
+
retryableStatuses: [408, 429, 500, 502, 503, 504, 510]
|
|
228
|
+
};
|
|
229
|
+
function classifyDomException(error) {
|
|
230
|
+
if (error.name === "AbortError") {
|
|
231
|
+
return { code: ErrorCode.TIMEOUT, message: "Request timed out", retryable: true };
|
|
232
|
+
}
|
|
233
|
+
return { code: ErrorCode.UNKNOWN_ERROR, message: error.message, retryable: false };
|
|
234
|
+
}
|
|
235
|
+
function classifyByErrorCode(error) {
|
|
236
|
+
const errCode = error.code;
|
|
237
|
+
if (!errCode) return null;
|
|
238
|
+
const networkErrorMessages = {
|
|
239
|
+
ECONNREFUSED: "Connection refused \u2014 service may be down",
|
|
240
|
+
ECONNRESET: "Connection was reset \u2014 please retry",
|
|
241
|
+
ECONNABORTED: "Connection aborted \u2014 please retry",
|
|
242
|
+
ENOTFOUND: "Service not reachable \u2014 check your network",
|
|
243
|
+
EPIPE: "Connection lost \u2014 please retry",
|
|
244
|
+
EAI_AGAIN: "DNS lookup failed \u2014 check your network"
|
|
245
|
+
};
|
|
246
|
+
if (errCode === "ECONNREFUSED" || errCode === "ENOTFOUND" || errCode === "ECONNRESET") {
|
|
247
|
+
return { code: ErrorCode.NETWORK_ERROR, message: networkErrorMessages[errCode] || "Network connection failed", retryable: true, cause: error.message };
|
|
248
|
+
}
|
|
249
|
+
if (errCode === "ECONNABORTED" || errCode === "ETIMEDOUT") {
|
|
250
|
+
return { code: ErrorCode.TIMEOUT, message: networkErrorMessages[errCode] || "Request timed out", retryable: true, cause: error.message };
|
|
251
|
+
}
|
|
252
|
+
return null;
|
|
253
|
+
}
|
|
254
|
+
function classifyByStatusCode(error) {
|
|
255
|
+
const status = error.response?.status || error.status || error.statusCode;
|
|
256
|
+
if (!status) return null;
|
|
257
|
+
return classifyHttpError(status, error.message || String(error));
|
|
258
|
+
}
|
|
259
|
+
function classifyByMessage(message) {
|
|
260
|
+
const lower = message.toLowerCase();
|
|
261
|
+
if (lower.includes("timeout") || lower.includes("timed out") || lower.includes("aborterror")) {
|
|
262
|
+
return { code: ErrorCode.TIMEOUT, message: "Request timed out", retryable: true, cause: message };
|
|
263
|
+
}
|
|
264
|
+
if (lower.includes("rate limit") || lower.includes("too many requests")) {
|
|
265
|
+
return { code: ErrorCode.RATE_LIMITED, message: "Rate limit exceeded", retryable: true, cause: message };
|
|
266
|
+
}
|
|
267
|
+
if (message.includes("API_KEY") || message.includes("api_key") || message.includes("Invalid API")) {
|
|
268
|
+
return { code: ErrorCode.AUTH_ERROR, message: "API key missing or invalid", retryable: false, cause: message };
|
|
269
|
+
}
|
|
270
|
+
if (message.includes("JSON") || message.includes("parse") || message.includes("Unexpected token")) {
|
|
271
|
+
return { code: ErrorCode.PARSE_ERROR, message: "Failed to parse response", retryable: false, cause: message };
|
|
272
|
+
}
|
|
273
|
+
return null;
|
|
274
|
+
}
|
|
275
|
+
function classifyFallback(message, cause) {
|
|
276
|
+
return {
|
|
277
|
+
code: ErrorCode.UNKNOWN_ERROR,
|
|
278
|
+
message,
|
|
279
|
+
retryable: false,
|
|
280
|
+
cause: cause ? String(cause) : void 0
|
|
281
|
+
};
|
|
282
|
+
}
|
|
283
|
+
function classifyError(error) {
|
|
284
|
+
if (error == null) {
|
|
285
|
+
return { code: ErrorCode.UNKNOWN_ERROR, message: "An unknown error occurred", retryable: false };
|
|
286
|
+
}
|
|
287
|
+
if (error instanceof DOMException) return classifyDomException(error);
|
|
288
|
+
if (!isErrorLike(error)) {
|
|
289
|
+
return { code: ErrorCode.UNKNOWN_ERROR, message: String(error), retryable: false };
|
|
290
|
+
}
|
|
291
|
+
return classifyByErrorCode(error) ?? classifyByStatusCode(error) ?? classifyByMessage(error.message ?? String(error)) ?? classifyFallback(error.message ?? String(error), error.cause);
|
|
292
|
+
}
|
|
293
|
+
function isErrorLike(value) {
|
|
294
|
+
return typeof value === "object" && value !== null;
|
|
295
|
+
}
|
|
296
|
+
function classifyHttpError(status, message) {
|
|
297
|
+
switch (status) {
|
|
298
|
+
case 400:
|
|
299
|
+
return { code: ErrorCode.INVALID_INPUT, message: "Bad request", retryable: false, statusCode: status };
|
|
300
|
+
case 401:
|
|
301
|
+
return { code: ErrorCode.AUTH_ERROR, message: "Invalid API key", retryable: false, statusCode: status };
|
|
302
|
+
case 403:
|
|
303
|
+
return { code: ErrorCode.QUOTA_EXCEEDED, message: "Access forbidden or quota exceeded", retryable: false, statusCode: status };
|
|
304
|
+
case 404:
|
|
305
|
+
return { code: ErrorCode.NOT_FOUND, message: "Resource not found", retryable: false, statusCode: status };
|
|
306
|
+
case 408:
|
|
307
|
+
return { code: ErrorCode.TIMEOUT, message: "Request timeout", retryable: true, statusCode: status };
|
|
308
|
+
case 429:
|
|
309
|
+
return { code: ErrorCode.RATE_LIMITED, message: "Rate limit exceeded", retryable: true, statusCode: status };
|
|
310
|
+
case 500:
|
|
311
|
+
return { code: ErrorCode.INTERNAL_ERROR, message: "Server error", retryable: true, statusCode: status };
|
|
312
|
+
case 502:
|
|
313
|
+
return { code: ErrorCode.SERVICE_UNAVAILABLE, message: "Bad gateway", retryable: true, statusCode: status };
|
|
314
|
+
case 503:
|
|
315
|
+
return { code: ErrorCode.SERVICE_UNAVAILABLE, message: "Service unavailable", retryable: true, statusCode: status };
|
|
316
|
+
case 504:
|
|
317
|
+
return { code: ErrorCode.TIMEOUT, message: "Gateway timeout", retryable: true, statusCode: status };
|
|
318
|
+
case 510:
|
|
319
|
+
return { code: ErrorCode.SERVICE_UNAVAILABLE, message: "Request canceled", retryable: true, statusCode: status };
|
|
320
|
+
default:
|
|
321
|
+
if (status >= 500) {
|
|
322
|
+
return { code: ErrorCode.SERVICE_UNAVAILABLE, message: `Server error: ${status}`, retryable: true, statusCode: status };
|
|
323
|
+
}
|
|
324
|
+
if (status >= 400) {
|
|
325
|
+
return { code: ErrorCode.INVALID_INPUT, message: `Client error: ${status}`, retryable: false, statusCode: status };
|
|
326
|
+
}
|
|
327
|
+
return { code: ErrorCode.UNKNOWN_ERROR, message: `HTTP ${status}: ${message}`, retryable: false, statusCode: status };
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
function calculateBackoff(attempt, options) {
|
|
331
|
+
const exponentialDelay = options.baseDelayMs * Math.pow(2, attempt);
|
|
332
|
+
const jitter = Math.random() * 0.3 * exponentialDelay;
|
|
333
|
+
return Math.min(exponentialDelay + jitter, options.maxDelayMs);
|
|
334
|
+
}
|
|
335
|
+
function sleep(ms, signal) {
|
|
336
|
+
return new Promise((resolve, reject) => {
|
|
337
|
+
if (signal?.aborted) {
|
|
338
|
+
reject(new DOMException("Aborted", "AbortError"));
|
|
339
|
+
return;
|
|
340
|
+
}
|
|
341
|
+
function onAbort() {
|
|
342
|
+
clearTimeout(timeout);
|
|
343
|
+
reject(new DOMException("Aborted", "AbortError"));
|
|
344
|
+
}
|
|
345
|
+
const timeout = setTimeout(() => {
|
|
346
|
+
if (signal) signal.removeEventListener("abort", onAbort);
|
|
347
|
+
resolve();
|
|
348
|
+
}, ms);
|
|
349
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
350
|
+
if (signal?.aborted) {
|
|
351
|
+
onAbort();
|
|
352
|
+
}
|
|
353
|
+
});
|
|
354
|
+
}
|
|
355
|
+
async function withStallProtection(fn, stallMs, maxAttempts = 2, label = "request") {
|
|
356
|
+
for (let attempt = 0; attempt < maxAttempts; attempt++) {
|
|
357
|
+
const controller = new AbortController();
|
|
358
|
+
let stallTimer;
|
|
359
|
+
const stallPromise = new Promise((_, reject) => {
|
|
360
|
+
stallTimer = setTimeout(() => {
|
|
361
|
+
controller.abort();
|
|
362
|
+
reject(Object.assign(new Error(`Service temporarily unavailable \u2014 no response received (attempt ${attempt + 1}/${maxAttempts})`), {
|
|
363
|
+
code: "ESTALLED",
|
|
364
|
+
retryable: attempt < maxAttempts - 1
|
|
365
|
+
}));
|
|
366
|
+
}, stallMs);
|
|
367
|
+
});
|
|
368
|
+
let fnPromise;
|
|
369
|
+
try {
|
|
370
|
+
fnPromise = fn(controller.signal);
|
|
371
|
+
const result = await Promise.race([fnPromise, stallPromise]);
|
|
372
|
+
clearTimeout(stallTimer);
|
|
373
|
+
return result;
|
|
374
|
+
} catch (err) {
|
|
375
|
+
fnPromise?.catch(() => {
|
|
376
|
+
});
|
|
377
|
+
clearTimeout(stallTimer);
|
|
378
|
+
const isStall = err instanceof Error && err.code === "ESTALLED";
|
|
379
|
+
if (isStall && attempt < maxAttempts - 1) {
|
|
380
|
+
const backoff = calculateBackoff(attempt, DEFAULT_RETRY_OPTIONS);
|
|
381
|
+
mcpLog("warning", `${label} stalled, retrying in ${backoff}ms (attempt ${attempt + 1})`, "stability");
|
|
382
|
+
await sleep(backoff);
|
|
383
|
+
continue;
|
|
384
|
+
}
|
|
385
|
+
throw err;
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
throw new Error(`${label} failed after ${maxAttempts} stall-protection attempts`);
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
// src/services/llm-processor.ts
|
|
392
|
+
var MAX_LLM_INPUT_CHARS = 5e5;
|
|
393
|
+
var MAX_PRIMARY_MODEL_INPUT_CHARS = 1e5;
|
|
394
|
+
var LLM_CLIENT_TIMEOUT_MS = 6e5;
|
|
395
|
+
var BACKOFF_JITTER_FACTOR = 0.3;
|
|
396
|
+
var LLM_STALL_TIMEOUT_MS = 75e3;
|
|
397
|
+
var LLM_REQUEST_DEADLINE_MS = 15e4;
|
|
398
|
+
var llmHealth = {
|
|
18
399
|
lastPlannerOk: false,
|
|
19
400
|
lastExtractorOk: false,
|
|
20
401
|
lastPlannerCheckedAt: null,
|
|
@@ -80,13 +461,13 @@ function _resetLLMHealthForTests() {
|
|
|
80
461
|
llmHealth.consecutivePlannerFailures = 0;
|
|
81
462
|
llmHealth.consecutiveExtractorFailures = 0;
|
|
82
463
|
}
|
|
83
|
-
|
|
464
|
+
var LLM_RETRY_CONFIG = {
|
|
84
465
|
maxRetries: 2,
|
|
85
466
|
baseDelayMs: 1e3,
|
|
86
467
|
maxDelayMs: 5e3
|
|
87
468
|
};
|
|
88
|
-
|
|
89
|
-
|
|
469
|
+
var FALLBACK_RETRY_COUNT = 3;
|
|
470
|
+
var RETRYABLE_LLM_ERROR_CODES = /* @__PURE__ */ new Set([
|
|
90
471
|
"rate_limit_exceeded",
|
|
91
472
|
"server_error",
|
|
92
473
|
"timeout",
|
|
@@ -95,7 +476,7 @@ const RETRYABLE_LLM_ERROR_CODES = /* @__PURE__ */ new Set([
|
|
|
95
476
|
function hasStatus(error) {
|
|
96
477
|
return typeof error === "object" && error !== null && "status" in error && typeof error.status === "number";
|
|
97
478
|
}
|
|
98
|
-
|
|
479
|
+
var llmClient = null;
|
|
99
480
|
function createLLMProcessor() {
|
|
100
481
|
if (!getCapabilities().llmExtraction) return null;
|
|
101
482
|
if (!llmClient) {
|
|
@@ -427,7 +808,7 @@ ${truncatedContent}`;
|
|
|
427
808
|
}
|
|
428
809
|
};
|
|
429
810
|
}
|
|
430
|
-
|
|
811
|
+
var MAX_CLASSIFICATION_URLS = 50;
|
|
431
812
|
async function classifySearchResults(rankedUrls, objective, totalQueries, processor, previousQueries = []) {
|
|
432
813
|
const urlsToClassify = rankedUrls.slice(0, MAX_CLASSIFICATION_URLS);
|
|
433
814
|
const STATIC_WEIGHTS = [30, 20, 15, 10, 8, 6, 5, 4, 3, 2];
|
|
@@ -595,7 +976,7 @@ RULES:
|
|
|
595
976
|
return { result: [], error: message };
|
|
596
977
|
}
|
|
597
978
|
}
|
|
598
|
-
|
|
979
|
+
var VALID_GOAL_CLASSES = /* @__PURE__ */ new Set([
|
|
599
980
|
"spec",
|
|
600
981
|
"bug",
|
|
601
982
|
"migration",
|
|
@@ -606,9 +987,9 @@ const VALID_GOAL_CLASSES = /* @__PURE__ */ new Set([
|
|
|
606
987
|
"product_launch",
|
|
607
988
|
"other"
|
|
608
989
|
]);
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
990
|
+
var VALID_FRESHNESS = /* @__PURE__ */ new Set(["days", "weeks", "months", "years"]);
|
|
991
|
+
var VALID_BRANCHES = /* @__PURE__ */ new Set(["reddit", "web", "both"]);
|
|
992
|
+
var VALID_STEP_TOOLS = /* @__PURE__ */ new Set(["raw-web-search", "smart-web-search", "raw-scrape-links", "smart-scrape-links"]);
|
|
612
993
|
function isStringArray(value) {
|
|
613
994
|
return Array.isArray(value) && value.every((v) => typeof v === "string");
|
|
614
995
|
}
|