mcp-researchpowerpack 3.6.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +635 -0
- package/dist/clients/reddit.d.ts +74 -0
- package/dist/clients/reddit.d.ts.map +1 -0
- package/dist/clients/reddit.js +305 -0
- package/dist/clients/reddit.js.map +1 -0
- package/dist/clients/research.d.ts +67 -0
- package/dist/clients/research.d.ts.map +1 -0
- package/dist/clients/research.js +252 -0
- package/dist/clients/research.js.map +1 -0
- package/dist/clients/scraper.d.ts +71 -0
- package/dist/clients/scraper.d.ts.map +1 -0
- package/dist/clients/scraper.js +321 -0
- package/dist/clients/scraper.js.map +1 -0
- package/dist/clients/search.d.ts +62 -0
- package/dist/clients/search.d.ts.map +1 -0
- package/dist/clients/search.js +219 -0
- package/dist/clients/search.js.map +1 -0
- package/dist/config/index.d.ts +62 -0
- package/dist/config/index.d.ts.map +1 -0
- package/dist/config/index.js +142 -0
- package/dist/config/index.js.map +1 -0
- package/dist/config/loader.d.ts +40 -0
- package/dist/config/loader.d.ts.map +1 -0
- package/dist/config/loader.js +305 -0
- package/dist/config/loader.js.map +1 -0
- package/dist/config/types.d.ts +81 -0
- package/dist/config/types.d.ts.map +1 -0
- package/dist/config/types.js +6 -0
- package/dist/config/types.js.map +1 -0
- package/dist/config/yaml/tools.yaml +130 -0
- package/dist/index.d.ts +7 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +271 -0
- package/dist/index.js.map +1 -0
- package/dist/schemas/deep-research.d.ts +64 -0
- package/dist/schemas/deep-research.d.ts.map +1 -0
- package/dist/schemas/deep-research.js +224 -0
- package/dist/schemas/deep-research.js.map +1 -0
- package/dist/schemas/scrape-links.d.ts +32 -0
- package/dist/schemas/scrape-links.d.ts.map +1 -0
- package/dist/schemas/scrape-links.js +34 -0
- package/dist/schemas/scrape-links.js.map +1 -0
- package/dist/schemas/web-search.d.ts +22 -0
- package/dist/schemas/web-search.d.ts.map +1 -0
- package/dist/schemas/web-search.js +21 -0
- package/dist/schemas/web-search.js.map +1 -0
- package/dist/services/file-attachment.d.ts +30 -0
- package/dist/services/file-attachment.d.ts.map +1 -0
- package/dist/services/file-attachment.js +199 -0
- package/dist/services/file-attachment.js.map +1 -0
- package/dist/services/llm-processor.d.ts +27 -0
- package/dist/services/llm-processor.d.ts.map +1 -0
- package/dist/services/llm-processor.js +179 -0
- package/dist/services/llm-processor.js.map +1 -0
- package/dist/services/markdown-cleaner.d.ts +8 -0
- package/dist/services/markdown-cleaner.d.ts.map +1 -0
- package/dist/services/markdown-cleaner.js +44 -0
- package/dist/services/markdown-cleaner.js.map +1 -0
- package/dist/tools/definitions.d.ts +16 -0
- package/dist/tools/definitions.d.ts.map +1 -0
- package/dist/tools/definitions.js +17 -0
- package/dist/tools/definitions.js.map +1 -0
- package/dist/tools/reddit.d.ts +14 -0
- package/dist/tools/reddit.d.ts.map +1 -0
- package/dist/tools/reddit.js +213 -0
- package/dist/tools/reddit.js.map +1 -0
- package/dist/tools/registry.d.ts +71 -0
- package/dist/tools/registry.d.ts.map +1 -0
- package/dist/tools/registry.js +242 -0
- package/dist/tools/registry.js.map +1 -0
- package/dist/tools/research.d.ts +14 -0
- package/dist/tools/research.d.ts.map +1 -0
- package/dist/tools/research.js +194 -0
- package/dist/tools/research.js.map +1 -0
- package/dist/tools/scrape.d.ts +14 -0
- package/dist/tools/scrape.d.ts.map +1 -0
- package/dist/tools/scrape.js +201 -0
- package/dist/tools/scrape.js.map +1 -0
- package/dist/tools/search.d.ts +10 -0
- package/dist/tools/search.d.ts.map +1 -0
- package/dist/tools/search.js +137 -0
- package/dist/tools/search.js.map +1 -0
- package/dist/tools/utils.d.ts +105 -0
- package/dist/tools/utils.d.ts.map +1 -0
- package/dist/tools/utils.js +159 -0
- package/dist/tools/utils.js.map +1 -0
- package/dist/utils/concurrency.d.ts +29 -0
- package/dist/utils/concurrency.d.ts.map +1 -0
- package/dist/utils/concurrency.js +73 -0
- package/dist/utils/concurrency.js.map +1 -0
- package/dist/utils/errors.d.ts +77 -0
- package/dist/utils/errors.d.ts.map +1 -0
- package/dist/utils/errors.js +335 -0
- package/dist/utils/errors.js.map +1 -0
- package/dist/utils/logger.d.ts +39 -0
- package/dist/utils/logger.d.ts.map +1 -0
- package/dist/utils/logger.js +57 -0
- package/dist/utils/logger.js.map +1 -0
- package/dist/utils/markdown-formatter.d.ts +5 -0
- package/dist/utils/markdown-formatter.d.ts.map +1 -0
- package/dist/utils/markdown-formatter.js +15 -0
- package/dist/utils/markdown-formatter.js.map +1 -0
- package/dist/utils/response.d.ts +88 -0
- package/dist/utils/response.d.ts.map +1 -0
- package/dist/utils/response.js +151 -0
- package/dist/utils/response.js.map +1 -0
- package/dist/utils/url-aggregator.d.ts +90 -0
- package/dist/utils/url-aggregator.d.ts.map +1 -0
- package/dist/utils/url-aggregator.js +502 -0
- package/dist/utils/url-aggregator.js.map +1 -0
- package/dist/version.d.ts +30 -0
- package/dist/version.d.ts.map +1 -0
- package/dist/version.js +60 -0
- package/dist/version.js.map +1 -0
- package/dist/worker.d.ts +17 -0
- package/dist/worker.d.ts.map +1 -0
- package/dist/worker.js +53 -0
- package/dist/worker.js.map +1 -0
- package/package.json +73 -0
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Web Search Client
|
|
3
|
+
* Generic interface for web search via Google (Serper implementation)
|
|
4
|
+
* Implements robust error handling that NEVER crashes
|
|
5
|
+
*/
|
|
6
|
+
import { parseEnv } from '../config/index.js';
|
|
7
|
+
import { classifyError, fetchWithTimeout, sleep, ErrorCode, } from '../utils/errors.js';
|
|
8
|
+
import { pMap } from '../utils/concurrency.js';
|
|
9
|
+
import { mcpLog } from '../utils/logger.js';
|
|
10
|
+
// Search retry configuration
|
|
11
|
+
const SEARCH_RETRY_CONFIG = {
|
|
12
|
+
maxRetries: 2,
|
|
13
|
+
baseDelayMs: 1000,
|
|
14
|
+
maxDelayMs: 10000,
|
|
15
|
+
timeoutMs: 30000,
|
|
16
|
+
};
|
|
17
|
+
const RETRYABLE_SEARCH_CODES = new Set([429, 500, 502, 503, 504]);
|
|
18
|
+
// Pre-compiled regex patterns for Reddit search
|
|
19
|
+
const REDDIT_SITE_REGEX = /site:\s*reddit\.com/i;
|
|
20
|
+
const REDDIT_SUBREDDIT_SUFFIX_REGEX = / : r\/\w+$/;
|
|
21
|
+
const REDDIT_SUFFIX_REGEX = / - Reddit$/;
|
|
22
|
+
export class SearchClient {
|
|
23
|
+
apiKey;
|
|
24
|
+
baseURL = 'https://google.serper.dev';
|
|
25
|
+
constructor(apiKey) {
|
|
26
|
+
const env = parseEnv();
|
|
27
|
+
this.apiKey = apiKey || env.SEARCH_API_KEY || '';
|
|
28
|
+
if (!this.apiKey) {
|
|
29
|
+
throw new Error('SERPER_API_KEY is required for search functionality');
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Calculate backoff delay
|
|
34
|
+
*/
|
|
35
|
+
calculateBackoff(attempt) {
|
|
36
|
+
const exponentialDelay = SEARCH_RETRY_CONFIG.baseDelayMs * Math.pow(2, attempt);
|
|
37
|
+
const jitter = Math.random() * 0.3 * exponentialDelay;
|
|
38
|
+
return Math.min(exponentialDelay + jitter, SEARCH_RETRY_CONFIG.maxDelayMs);
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Check if error is retryable
|
|
42
|
+
*/
|
|
43
|
+
isRetryable(status, error) {
|
|
44
|
+
if (status && RETRYABLE_SEARCH_CODES.has(status))
|
|
45
|
+
return true;
|
|
46
|
+
const message = error?.message?.toLowerCase() || '';
|
|
47
|
+
return message.includes('timeout') || message.includes('rate limit') || message.includes('connection');
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Search multiple keywords in parallel
|
|
51
|
+
* NEVER throws - always returns a valid response
|
|
52
|
+
*/
|
|
53
|
+
async searchMultiple(keywords) {
|
|
54
|
+
const startTime = Date.now();
|
|
55
|
+
if (keywords.length === 0) {
|
|
56
|
+
return {
|
|
57
|
+
searches: [],
|
|
58
|
+
totalKeywords: 0,
|
|
59
|
+
executionTime: 0,
|
|
60
|
+
error: { code: ErrorCode.INVALID_INPUT, message: 'No keywords provided', retryable: false },
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
let lastError;
|
|
64
|
+
for (let attempt = 0; attempt <= SEARCH_RETRY_CONFIG.maxRetries; attempt++) {
|
|
65
|
+
try {
|
|
66
|
+
if (attempt > 0) {
|
|
67
|
+
mcpLog('warning', `Retry attempt ${attempt}/${SEARCH_RETRY_CONFIG.maxRetries}`, 'search');
|
|
68
|
+
}
|
|
69
|
+
const searchQueries = keywords.map(keyword => ({ q: keyword }));
|
|
70
|
+
const response = await fetchWithTimeout(`${this.baseURL}/search`, {
|
|
71
|
+
method: 'POST',
|
|
72
|
+
headers: {
|
|
73
|
+
'X-API-KEY': this.apiKey,
|
|
74
|
+
'Content-Type': 'application/json',
|
|
75
|
+
},
|
|
76
|
+
body: JSON.stringify(searchQueries),
|
|
77
|
+
timeoutMs: SEARCH_RETRY_CONFIG.timeoutMs,
|
|
78
|
+
});
|
|
79
|
+
if (!response.ok) {
|
|
80
|
+
const errorText = await response.text().catch(() => '');
|
|
81
|
+
lastError = classifyError({ status: response.status, message: errorText });
|
|
82
|
+
if (this.isRetryable(response.status) && attempt < SEARCH_RETRY_CONFIG.maxRetries) {
|
|
83
|
+
const delayMs = this.calculateBackoff(attempt);
|
|
84
|
+
mcpLog('warning', `API returned ${response.status}, retrying in ${delayMs}ms...`, 'search');
|
|
85
|
+
await sleep(delayMs);
|
|
86
|
+
continue;
|
|
87
|
+
}
|
|
88
|
+
// Return partial result with error
|
|
89
|
+
return {
|
|
90
|
+
searches: [],
|
|
91
|
+
totalKeywords: keywords.length,
|
|
92
|
+
executionTime: Date.now() - startTime,
|
|
93
|
+
error: lastError,
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
// Parse response safely
|
|
97
|
+
let data;
|
|
98
|
+
try {
|
|
99
|
+
data = await response.json();
|
|
100
|
+
}
|
|
101
|
+
catch (parseError) {
|
|
102
|
+
return {
|
|
103
|
+
searches: [],
|
|
104
|
+
totalKeywords: keywords.length,
|
|
105
|
+
executionTime: Date.now() - startTime,
|
|
106
|
+
error: { code: ErrorCode.PARSE_ERROR, message: 'Failed to parse search response', retryable: false },
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
const responses = Array.isArray(data) ? data : [data];
|
|
110
|
+
const searches = responses.map((resp, index) => {
|
|
111
|
+
try {
|
|
112
|
+
const organic = (resp.organic || []);
|
|
113
|
+
const results = organic.map((item, idx) => ({
|
|
114
|
+
title: item.title || 'No title',
|
|
115
|
+
link: item.link || '#',
|
|
116
|
+
snippet: item.snippet || '',
|
|
117
|
+
date: item.date,
|
|
118
|
+
position: item.position || idx + 1,
|
|
119
|
+
}));
|
|
120
|
+
const searchInfo = resp.searchInformation;
|
|
121
|
+
const totalResults = searchInfo?.totalResults
|
|
122
|
+
? parseInt(String(searchInfo.totalResults).replace(/,/g, ''), 10)
|
|
123
|
+
: results.length;
|
|
124
|
+
const relatedSearches = (resp.relatedSearches || []);
|
|
125
|
+
const related = relatedSearches.map((r) => r.query || '');
|
|
126
|
+
return { keyword: keywords[index] || '', results, totalResults, related };
|
|
127
|
+
}
|
|
128
|
+
catch {
|
|
129
|
+
// Return empty result for this keyword on parse error
|
|
130
|
+
return { keyword: keywords[index] || '', results: [], totalResults: 0, related: [] };
|
|
131
|
+
}
|
|
132
|
+
});
|
|
133
|
+
return { searches, totalKeywords: keywords.length, executionTime: Date.now() - startTime };
|
|
134
|
+
}
|
|
135
|
+
catch (error) {
|
|
136
|
+
lastError = classifyError(error);
|
|
137
|
+
if (this.isRetryable(undefined, error) && attempt < SEARCH_RETRY_CONFIG.maxRetries) {
|
|
138
|
+
const delayMs = this.calculateBackoff(attempt);
|
|
139
|
+
mcpLog('warning', `${lastError.code}: ${lastError.message}, retrying in ${delayMs}ms...`, 'search');
|
|
140
|
+
await sleep(delayMs);
|
|
141
|
+
continue;
|
|
142
|
+
}
|
|
143
|
+
break;
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
// All retries failed
|
|
147
|
+
return {
|
|
148
|
+
searches: [],
|
|
149
|
+
totalKeywords: keywords.length,
|
|
150
|
+
executionTime: Date.now() - startTime,
|
|
151
|
+
error: lastError || { code: ErrorCode.UNKNOWN_ERROR, message: 'Search failed', retryable: false },
|
|
152
|
+
};
|
|
153
|
+
}
|
|
154
|
+
/**
|
|
155
|
+
* Search Reddit via Google (adds site:reddit.com automatically)
|
|
156
|
+
* NEVER throws - returns empty array on failure
|
|
157
|
+
*/
|
|
158
|
+
async searchReddit(query, dateAfter) {
|
|
159
|
+
if (!query?.trim()) {
|
|
160
|
+
return [];
|
|
161
|
+
}
|
|
162
|
+
let q = REDDIT_SITE_REGEX.test(query) ? query : `${query} site:reddit.com`;
|
|
163
|
+
if (dateAfter) {
|
|
164
|
+
q += ` after:${dateAfter}`;
|
|
165
|
+
}
|
|
166
|
+
for (let attempt = 0; attempt <= SEARCH_RETRY_CONFIG.maxRetries; attempt++) {
|
|
167
|
+
try {
|
|
168
|
+
const res = await fetchWithTimeout(`${this.baseURL}/search`, {
|
|
169
|
+
method: 'POST',
|
|
170
|
+
headers: { 'X-API-KEY': this.apiKey, 'Content-Type': 'application/json' },
|
|
171
|
+
body: JSON.stringify({ q, num: 10 }),
|
|
172
|
+
timeoutMs: SEARCH_RETRY_CONFIG.timeoutMs,
|
|
173
|
+
});
|
|
174
|
+
if (!res.ok) {
|
|
175
|
+
if (this.isRetryable(res.status) && attempt < SEARCH_RETRY_CONFIG.maxRetries) {
|
|
176
|
+
const delayMs = this.calculateBackoff(attempt);
|
|
177
|
+
mcpLog('warning', `Reddit search ${res.status}, retrying in ${delayMs}ms...`, 'search');
|
|
178
|
+
await sleep(delayMs);
|
|
179
|
+
continue;
|
|
180
|
+
}
|
|
181
|
+
mcpLog('error', `Reddit search failed with status ${res.status}`, 'search');
|
|
182
|
+
return [];
|
|
183
|
+
}
|
|
184
|
+
const data = await res.json();
|
|
185
|
+
return (data.organic || []).map((r) => ({
|
|
186
|
+
title: (r.title || '').replace(REDDIT_SUBREDDIT_SUFFIX_REGEX, '').replace(REDDIT_SUFFIX_REGEX, ''),
|
|
187
|
+
url: r.link || '',
|
|
188
|
+
snippet: r.snippet || '',
|
|
189
|
+
date: r.date,
|
|
190
|
+
}));
|
|
191
|
+
}
|
|
192
|
+
catch (error) {
|
|
193
|
+
const err = classifyError(error);
|
|
194
|
+
if (this.isRetryable(undefined, error) && attempt < SEARCH_RETRY_CONFIG.maxRetries) {
|
|
195
|
+
const delayMs = this.calculateBackoff(attempt);
|
|
196
|
+
mcpLog('warning', `Reddit search ${err.code}, retrying in ${delayMs}ms...`, 'search');
|
|
197
|
+
await sleep(delayMs);
|
|
198
|
+
continue;
|
|
199
|
+
}
|
|
200
|
+
mcpLog('error', `Reddit search failed: ${err.message}`, 'search');
|
|
201
|
+
return [];
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
return [];
|
|
205
|
+
}
|
|
206
|
+
/**
|
|
207
|
+
* Search Reddit with multiple queries (bounded concurrency)
|
|
208
|
+
* NEVER throws - searchReddit never throws, pMap preserves order
|
|
209
|
+
*/
|
|
210
|
+
async searchRedditMultiple(queries, dateAfter) {
|
|
211
|
+
if (queries.length === 0) {
|
|
212
|
+
return new Map();
|
|
213
|
+
}
|
|
214
|
+
// Limit to 8 concurrent Serper API calls to prevent CPU spikes & rate limits
|
|
215
|
+
const results = await pMap(queries, q => this.searchReddit(q, dateAfter), 8);
|
|
216
|
+
return new Map(queries.map((q, i) => [q, results[i] || []]));
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
//# sourceMappingURL=search.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"search.js","sourceRoot":"","sources":["../../src/clients/search.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EACL,aAAa,EACb,gBAAgB,EAChB,KAAK,EACL,SAAS,GAEV,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,IAAI,EAAE,MAAM,yBAAyB,CAAC;AAC/C,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAgC5C,6BAA6B;AAC7B,MAAM,mBAAmB,GAAG;IAC1B,UAAU,EAAE,CAAC;IACb,WAAW,EAAE,IAAI;IACjB,UAAU,EAAE,KAAK;IACjB,SAAS,EAAE,KAAK;CACR,CAAC;AAEX,MAAM,sBAAsB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;AAElE,gDAAgD;AAChD,MAAM,iBAAiB,GAAG,sBAAsB,CAAC;AACjD,MAAM,6BAA6B,GAAG,YAAY,CAAC;AACnD,MAAM,mBAAmB,GAAG,YAAY,CAAC;AAEzC,MAAM,OAAO,YAAY;IACf,MAAM,CAAS;IACf,OAAO,GAAG,2BAA2B,CAAC;IAE9C,YAAY,MAAe;QACzB,MAAM,GAAG,GAAG,QAAQ,EAAE,CAAC;QACvB,IAAI,CAAC,MAAM,GAAG,MAAM,IAAI,GAAG,CAAC,cAAc,IAAI,EAAE,CAAC;QAEjD,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;QACzE,CAAC;IACH,CAAC;IAED;;OAEG;IACK,gBAAgB,CAAC,OAAe;QACtC,MAAM,gBAAgB,GAAG,mBAAmB,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;QAChF,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,GAAG,GAAG,gBAAgB,CAAC;QACtD,OAAO,IAAI,CAAC,GAAG,CAAC,gBAAgB,GAAG,MAAM,EAAE,mBAAmB,CAAC,UAAU,CAAC,CAAC;IAC7E,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,MAAe,EAAE,KAAe;QAClD,IAAI,MAAM,IAAI,sBAAsB,CAAC,GAAG,CAAC,MAAM,CAAC;YAAE,OAAO,IAAI,CAAC;QAE9D,MAAM,OAAO,GAAI,KAA8B,EAAE,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;QAC9E,OAAO,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;IACzG,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,cAAc,CAAC,QAAkB;QACrC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE7B,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,OAAO;gBACL,QAAQ,EAAE,EAAE;gBACZ,aAAa,EAAE,CAAC;gBAChB,aAAa,EAAE,CAAC;gBAChB,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,CAAC,aAAa,EAAE,OAAO,EAAE,sBAAsB,EAAE,SAAS,EAAE,KAAK,EAAE;aAC5F,CAAC;QACJ,CAAC;QAED,IAAI,SAAsC,CAAC;QAE3C,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,mBAAmB,CAAC,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;YAC3E,IAAI,CAAC;gBACH,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;oBAChB,MAAM,CAAC,SAAS,EAAE,iBAAiB,OAAO,IAAI,mBAAmB,CAAC,UAAU,EAAE,EAAE,QAAQ,CAAC,CAAC;gBAC5F,CAAC;gBAED,MAAM,aAAa,GAAG,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;gBAEhE,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,GAAG,IAAI,CAAC,OAAO,SAAS,EAAE;oBAChE,MAAM,EAAE,MAAM;oBACd,OAAO,EAAE;wBACP,WAAW,EAAE,IAAI,CAAC,MAAM;wBACxB,cAAc,EAAE,kBAAkB;qBACnC;oBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC;oBACnC,SAAS,EAAE,mBAAmB,CAAC,SAAS;iBACzC,CAAC,CAAC;gBAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;oBACjB,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;oBACxD,SAAS,GAAG,aAAa,CAAC,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC;oBAE3E,IAAI,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,OAAO,GAAG,mBAAmB,CAAC,UAAU,EAAE,CAAC;wBAClF,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;wBAC/C,MAAM,CAAC,SAAS,EAAE,gBAAgB,QAAQ,CAAC,MAAM,iBAAiB,OAAO,OAAO,EAAE,QAAQ,CAAC,CAAC;wBAC5F,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;wBACrB,SAAS;oBACX,CAAC;oBAED,mCAAmC;oBACnC,OAAO;wBACL,QAAQ,EAAE,EAAE;wBACZ,aAAa,EAAE,QAAQ,CAAC,MAAM;wBAC9B,aAAa,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;wBACrC,KAAK,EAAE,SAAS;qBACjB,CAAC;gBACJ,CAAC;gBAED,wBAAwB;gBACxB,IAAI,IAAa,CAAC;gBAClB,IAAI,CAAC;oBACH,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;gBAC/B,CAAC;gBAAC,OAAO,UAAU,EAAE,CAAC;oBACpB,OAAO;wBACL,QAAQ,EAAE,EAAE;wBACZ,aAAa,EAAE,QAAQ,CAAC,MAAM;wBAC9B,aAAa,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;wBACrC,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,CAAC,WAAW,EAAE,OAAO,EAAE,iCAAiC,EAAE,SAAS,EAAE,KAAK,EAAE;qBACrG,CAAC;gBACJ,CAAC;gBAED,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBAEtD,MAAM,QAAQ,GAA0B,SAAS,CAAC,GAAG,CAAC,CAAC,IAA6B,EAAE,KAAa,EAAE,EAAE;oBACrG,IAAI,CAAC;wBACH,MAAM,OAAO,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAmC,CAAC;wBACvE,MAAM,OAAO,GAAmB,OAAO,CAAC,GAAG,CAAC,CAAC,IAA6B,EAAE,GAAW,EAAE,EAAE,CAAC,CAAC;4BAC3F,KAAK,EAAG,IAAI,CAAC,KAAgB,IAAI,UAAU;4BAC3C,IAAI,EAAG,IAAI,CAAC,IAAe,IAAI,GAAG;4BAClC,OAAO,EAAG,IAAI,CAAC,OAAkB,IAAI,EAAE;4BACvC,IAAI,EAAE,IAAI,CAAC,IAA0B;4BACrC,QAAQ,EAAG,IAAI,CAAC,QAAmB,IAAI,GAAG,GAAG,CAAC;yBAC/C,CAAC,CAAC,CAAC;wBAEJ,MAAM,UAAU,GAAG,IAAI,CAAC,iBAAwD,CAAC;wBACjF,MAAM,YAAY,GAAG,UAAU,EAAE,YAAY;4BAC3C,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;4BACjE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;wBAEnB,MAAM,eAAe,GAAG,CAAC,IAAI,CAAC,eAAe,IAAI,EAAE,CAAmC,CAAC;wBACvF,MAAM,OAAO,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC,CAA0B,EAAE,EAAE,CAAE,CAAC,CAAC,KAAgB,IAAI,EAAE,CAAC,CAAC;wBAE/F,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC;oBAC5E,CAAC;oBAAC,MAAM,CAAC;wBACP,sDAAsD;wBACtD,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;oBACvF,CAAC;gBACH,CAAC,CAAC,CAAC;gBAEH,OAAO,EAAE,QAAQ,EAAE,aAAa,EAAE,QAAQ,CAAC,MAAM,EAAE,aAAa,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,EAAE,CAAC;YAE7F,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,SAAS,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;gBAEjC,IAAI,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,OAAO,GAAG,mBAAmB,CAAC,UAAU,EAAE,CAAC;oBACnF,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;oBAC/C,MAAM,CAAC,SAAS,EAAE,GAAG,SAAS,CAAC,IAAI,KAAK,SAAS,CAAC,OAAO,iBAAiB,OAAO,OAAO,EAAE,QAAQ,CAAC,CAAC;oBACpG,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;oBACrB,SAAS;gBACX,CAAC;gBAED,MAAM;YACR,CAAC;QACH,CAAC;QAED,qBAAqB;QACrB,OAAO;YACL,QAAQ,EAAE,EAAE;YACZ,aAAa,EAAE,QAAQ,CAAC,MAAM;YAC9B,aAAa,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;YACrC,KAAK,EAAE,SAAS,IAAI,EAAE,IAAI,EAAE,SAAS,CAAC,aAAa,EAAE,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,KAAK,EAAE;SAClG,CAAC;IACJ,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,YAAY,CAAC,KAAa,EAAE,SAAkB;QAClD,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC;YACnB,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,IAAI,CAAC,GAAG,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,kBAAkB,CAAC;QAE3E,IAAI,SAAS,EAAE,CAAC;YACd,CAAC,IAAI,UAAU,SAAS,EAAE,CAAC;QAC7B,CAAC;QAED,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,mBAAmB,CAAC,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;YAC3E,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,MAAM,gBAAgB,CAAC,GAAG,IAAI,CAAC,OAAO,SAAS,EAAE;oBAC3D,MAAM,EAAE,MAAM;oBACd,OAAO,EAAE,EAAE,WAAW,EAAE,IAAI,CAAC,MAAM,EAAE,cAAc,EAAE,kBAAkB,EAAE;oBACzE,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC;oBACpC,SAAS,EAAE,mBAAmB,CAAC,SAAS;iBACzC,CAAC,CAAC;gBAEH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;oBACZ,IAAI,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,OAAO,GAAG,mBAAmB,CAAC,UAAU,EAAE,CAAC;wBAC7E,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;wBAC/C,MAAM,CAAC,SAAS,EAAE,iBAAiB,GAAG,CAAC,MAAM,iBAAiB,OAAO,OAAO,EAAE,QAAQ,CAAC,CAAC;wBACxF,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;wBACrB,SAAS;oBACX,CAAC;oBACD,MAAM,CAAC,OAAO,EAAE,oCAAoC,GAAG,CAAC,MAAM,EAAE,EAAE,QAAQ,CAAC,CAAC;oBAC5E,OAAO,EAAE,CAAC;gBACZ,CAAC;gBAED,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAA0F,CAAC;gBACtH,OAAO,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;oBACtC,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,6BAA6B,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,mBAAmB,EAAE,EAAE,CAAC;oBAClG,GAAG,EAAE,CAAC,CAAC,IAAI,IAAI,EAAE;oBACjB,OAAO,EAAE,CAAC,CAAC,OAAO,IAAI,EAAE;oBACxB,IAAI,EAAE,CAAC,CAAC,IAAI;iBACb,CAAC,CAAC,CAAC;YAEN,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,GAAG,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;gBACjC,IAAI,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,OAAO,GAAG,mBAAmB,CAAC,UAAU,EAAE,CAAC;oBACnF,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;oBAC/C,MAAM,CAAC,SAAS,EAAE,iBAAiB,GAAG,CAAC,IAAI,iBAAiB,OAAO,OAAO,EAAE,QAAQ,CAAC,CAAC;oBACtF,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;oBACrB,SAAS;gBACX,CAAC;gBACD,MAAM,CAAC,OAAO,EAAE,yBAAyB,GAAG,CAAC,OAAO,EAAE,EAAE,QAAQ,CAAC,CAAC;gBAClE,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC;QAED,OAAO,EAAE,CAAC;IACZ,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,oBAAoB,CAAC,OAAiB,EAAE,SAAkB;QAC9D,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,IAAI,GAAG,EAAE,CAAC;QACnB,CAAC;QAED,6EAA6E;QAC7E,MAAM,OAAO,GAAG,MAAM,IAAI,CACxB,OAAO,EACP,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,EAAE,SAAS,CAAC,EACpC,CAAC,CACF,CAAC;QAEF,OAAO,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAC/D,CAAC;CACF"}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Consolidated configuration
|
|
3
|
+
* All environment variables, constants, and LLM config in one place
|
|
4
|
+
*/
|
|
5
|
+
interface EnvConfig {
|
|
6
|
+
SCRAPER_API_KEY: string;
|
|
7
|
+
SEARCH_API_KEY: string | undefined;
|
|
8
|
+
REDDIT_CLIENT_ID: string | undefined;
|
|
9
|
+
REDDIT_CLIENT_SECRET: string | undefined;
|
|
10
|
+
}
|
|
11
|
+
export declare function parseEnv(): EnvConfig;
|
|
12
|
+
export declare const RESEARCH: {
|
|
13
|
+
readonly BASE_URL: string;
|
|
14
|
+
readonly MODEL: string;
|
|
15
|
+
readonly FALLBACK_MODEL: string;
|
|
16
|
+
readonly API_KEY: string;
|
|
17
|
+
readonly TIMEOUT_MS: number;
|
|
18
|
+
readonly REASONING_EFFORT: "low" | "medium" | "high";
|
|
19
|
+
readonly MAX_URLS: number;
|
|
20
|
+
};
|
|
21
|
+
export declare const SERVER: {
|
|
22
|
+
readonly NAME: string;
|
|
23
|
+
readonly VERSION: string;
|
|
24
|
+
readonly DESCRIPTION: string;
|
|
25
|
+
};
|
|
26
|
+
export interface Capabilities {
|
|
27
|
+
reddit: boolean;
|
|
28
|
+
search: boolean;
|
|
29
|
+
scraping: boolean;
|
|
30
|
+
deepResearch: boolean;
|
|
31
|
+
llmExtraction: boolean;
|
|
32
|
+
}
|
|
33
|
+
export declare function getCapabilities(): Capabilities;
|
|
34
|
+
export declare function getMissingEnvMessage(capability: keyof Capabilities): string;
|
|
35
|
+
export declare const SCRAPER: {
|
|
36
|
+
readonly MAX_CONCURRENT: 30;
|
|
37
|
+
readonly BATCH_SIZE: 30;
|
|
38
|
+
readonly MAX_TOKENS_BUDGET: 32000;
|
|
39
|
+
readonly MIN_URLS: 3;
|
|
40
|
+
readonly MAX_URLS: 50;
|
|
41
|
+
readonly RETRY_COUNT: 3;
|
|
42
|
+
readonly RETRY_DELAYS: readonly [2000, 4000, 8000];
|
|
43
|
+
readonly EXTRACTION_SUFFIX: "Try to answer this information as comprehensive as possible while keeping info density super high without adding unnecessary words but satisfy the scope defined by previous instructions even more.";
|
|
44
|
+
};
|
|
45
|
+
export declare const REDDIT: {
|
|
46
|
+
readonly MAX_CONCURRENT: 10;
|
|
47
|
+
readonly BATCH_SIZE: 10;
|
|
48
|
+
readonly MAX_COMMENT_BUDGET: 1000;
|
|
49
|
+
readonly MAX_COMMENTS_PER_POST: 200;
|
|
50
|
+
readonly MIN_POSTS: 2;
|
|
51
|
+
readonly MAX_POSTS: 50;
|
|
52
|
+
readonly RETRY_COUNT: 5;
|
|
53
|
+
readonly RETRY_DELAYS: readonly [2000, 4000, 8000, 16000, 32000];
|
|
54
|
+
};
|
|
55
|
+
export declare const CTR_WEIGHTS: Record<number, number>;
|
|
56
|
+
export declare const LLM_EXTRACTION: {
|
|
57
|
+
readonly MODEL: string;
|
|
58
|
+
readonly MAX_TOKENS: 8000;
|
|
59
|
+
readonly ENABLE_REASONING: boolean;
|
|
60
|
+
};
|
|
61
|
+
export {};
|
|
62
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAoDH,UAAU,SAAS;IACjB,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,GAAG,SAAS,CAAC;IACnC,gBAAgB,EAAE,MAAM,GAAG,SAAS,CAAC;IACrC,oBAAoB,EAAE,MAAM,GAAG,SAAS,CAAC;CAC1C;AAID,wBAAgB,QAAQ,IAAI,SAAS,CASpC;AAMD,eAAO,MAAM,QAAQ;;;;;;;;CAUX,CAAC;AAQX,eAAO,MAAM,MAAM;;;;CAIT,CAAC;AAMX,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,OAAO,CAAC;IAChB,MAAM,EAAE,OAAO,CAAC;IAChB,QAAQ,EAAE,OAAO,CAAC;IAClB,YAAY,EAAE,OAAO,CAAC;IACtB,aAAa,EAAE,OAAO,CAAC;CACxB;AAED,wBAAgB,eAAe,IAAI,YAAY,CAS9C;AAED,wBAAgB,oBAAoB,CAAC,UAAU,EAAE,MAAM,YAAY,GAAG,MAAM,CAS3E;AAMD,eAAO,MAAM,OAAO;;;;;;;;;CASV,CAAC;AAMX,eAAO,MAAM,MAAM;;;;;;;;;CAST,CAAC;AAMX,eAAO,MAAM,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAWrC,CAAC;AAMX,eAAO,MAAM,cAAc;;;;CAIjB,CAAC"}
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Consolidated configuration
|
|
3
|
+
* All environment variables, constants, and LLM config in one place
|
|
4
|
+
*/
|
|
5
|
+
import { VERSION, PACKAGE_NAME, PACKAGE_DESCRIPTION } from '../version.js';
|
|
6
|
+
// Import version utilities (not re-exported - use directly from version.ts if needed externally)
|
|
7
|
+
// ============================================================================
|
|
8
|
+
// Safe Integer Parsing Helper
|
|
9
|
+
// ============================================================================
|
|
10
|
+
/**
|
|
11
|
+
* Safely parse an integer from environment variable with bounds checking
|
|
12
|
+
* @param value - The string value to parse (from process.env)
|
|
13
|
+
* @param defaultVal - Default value if parsing fails or value is undefined
|
|
14
|
+
* @param min - Minimum allowed value (clamped if below)
|
|
15
|
+
* @param max - Maximum allowed value (clamped if above)
|
|
16
|
+
* @returns Parsed integer within bounds, or default value
|
|
17
|
+
*/
|
|
18
|
+
function safeParseInt(value, defaultVal, min, max) {
|
|
19
|
+
if (!value) {
|
|
20
|
+
return defaultVal;
|
|
21
|
+
}
|
|
22
|
+
const parsed = parseInt(value, 10);
|
|
23
|
+
if (isNaN(parsed)) {
|
|
24
|
+
console.warn(`[Config] Invalid number "${value}", using default ${defaultVal}`);
|
|
25
|
+
return defaultVal;
|
|
26
|
+
}
|
|
27
|
+
if (parsed < min) {
|
|
28
|
+
console.warn(`[Config] Value ${parsed} below minimum ${min}, clamping to ${min}`);
|
|
29
|
+
return min;
|
|
30
|
+
}
|
|
31
|
+
if (parsed > max) {
|
|
32
|
+
console.warn(`[Config] Value ${parsed} above maximum ${max}, clamping to ${max}`);
|
|
33
|
+
return max;
|
|
34
|
+
}
|
|
35
|
+
return parsed;
|
|
36
|
+
}
|
|
37
|
+
let cachedEnv = null;
|
|
38
|
+
export function parseEnv() {
|
|
39
|
+
if (cachedEnv)
|
|
40
|
+
return cachedEnv;
|
|
41
|
+
cachedEnv = {
|
|
42
|
+
SCRAPER_API_KEY: process.env.SCRAPEDO_API_KEY || '',
|
|
43
|
+
SEARCH_API_KEY: process.env.SERPER_API_KEY || undefined,
|
|
44
|
+
REDDIT_CLIENT_ID: process.env.REDDIT_CLIENT_ID || undefined,
|
|
45
|
+
REDDIT_CLIENT_SECRET: process.env.REDDIT_CLIENT_SECRET || undefined,
|
|
46
|
+
};
|
|
47
|
+
return cachedEnv;
|
|
48
|
+
}
|
|
49
|
+
// ============================================================================
|
|
50
|
+
// Research API Configuration
|
|
51
|
+
// ============================================================================
|
|
52
|
+
export const RESEARCH = {
|
|
53
|
+
BASE_URL: process.env.OPENROUTER_BASE_URL || 'https://openrouter.ai/api/v1',
|
|
54
|
+
MODEL: process.env.RESEARCH_MODEL || 'x-ai/grok-4-fast',
|
|
55
|
+
FALLBACK_MODEL: process.env.RESEARCH_FALLBACK_MODEL || 'google/gemini-2.5-flash',
|
|
56
|
+
API_KEY: process.env.OPENROUTER_API_KEY || '',
|
|
57
|
+
// Timeout: min 1s, max 1hr, default 30min
|
|
58
|
+
TIMEOUT_MS: safeParseInt(process.env.API_TIMEOUT_MS, 1800000, 1000, 3600000),
|
|
59
|
+
REASONING_EFFORT: process.env.DEFAULT_REASONING_EFFORT || 'high',
|
|
60
|
+
// Max URLs in search results: min 10, max 200, default 100
|
|
61
|
+
MAX_URLS: safeParseInt(process.env.DEFAULT_MAX_URLS, 100, 10, 200),
|
|
62
|
+
};
|
|
63
|
+
// ============================================================================
|
|
64
|
+
// MCP Server Configuration
|
|
65
|
+
// ============================================================================
|
|
66
|
+
// Version is now automatically read from package.json via version.ts
|
|
67
|
+
// No need to manually update version strings anymore!
|
|
68
|
+
export const SERVER = {
|
|
69
|
+
NAME: PACKAGE_NAME,
|
|
70
|
+
VERSION: VERSION,
|
|
71
|
+
DESCRIPTION: PACKAGE_DESCRIPTION,
|
|
72
|
+
};
|
|
73
|
+
export function getCapabilities() {
|
|
74
|
+
const env = parseEnv();
|
|
75
|
+
return {
|
|
76
|
+
reddit: !!(env.REDDIT_CLIENT_ID && env.REDDIT_CLIENT_SECRET),
|
|
77
|
+
search: !!env.SEARCH_API_KEY,
|
|
78
|
+
scraping: !!env.SCRAPER_API_KEY,
|
|
79
|
+
deepResearch: !!RESEARCH.API_KEY,
|
|
80
|
+
llmExtraction: !!RESEARCH.API_KEY, // Reuses OPENROUTER for LLM extraction
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
export function getMissingEnvMessage(capability) {
|
|
84
|
+
const messages = {
|
|
85
|
+
reddit: 'ā **Reddit tools unavailable.** Set `REDDIT_CLIENT_ID` and `REDDIT_CLIENT_SECRET` to enable.\n\nš Create a Reddit app at: https://www.reddit.com/prefs/apps (select "script" type)',
|
|
86
|
+
search: 'ā **Search unavailable.** Set `SERPER_API_KEY` to enable web search and Reddit search.\n\nš Get your free API key at: https://serper.dev (2,500 free queries)',
|
|
87
|
+
scraping: 'ā **Web scraping unavailable.** Set `SCRAPEDO_API_KEY` to enable URL content extraction.\n\nš Sign up at: https://scrape.do (1,000 free credits)',
|
|
88
|
+
deepResearch: 'ā **Deep research unavailable.** Set `OPENROUTER_API_KEY` to enable AI-powered research.\n\nš Get your API key at: https://openrouter.ai/keys',
|
|
89
|
+
llmExtraction: 'ā ļø **AI extraction disabled.** The `use_llm` and `what_to_extract` features require `OPENROUTER_API_KEY`.\n\nScraping will work but without intelligent content filtering.',
|
|
90
|
+
};
|
|
91
|
+
return messages[capability];
|
|
92
|
+
}
|
|
93
|
+
// ============================================================================
|
|
94
|
+
// Scraper Configuration (Scrape.do implementation)
|
|
95
|
+
// ============================================================================
|
|
96
|
+
export const SCRAPER = {
|
|
97
|
+
MAX_CONCURRENT: 30,
|
|
98
|
+
BATCH_SIZE: 30,
|
|
99
|
+
MAX_TOKENS_BUDGET: 32000,
|
|
100
|
+
MIN_URLS: 3,
|
|
101
|
+
MAX_URLS: 50,
|
|
102
|
+
RETRY_COUNT: 3,
|
|
103
|
+
RETRY_DELAYS: [2000, 4000, 8000],
|
|
104
|
+
EXTRACTION_SUFFIX: 'Try to answer this information as comprehensive as possible while keeping info density super high without adding unnecessary words but satisfy the scope defined by previous instructions even more.',
|
|
105
|
+
};
|
|
106
|
+
// ============================================================================
|
|
107
|
+
// Reddit Configuration
|
|
108
|
+
// ============================================================================
|
|
109
|
+
export const REDDIT = {
|
|
110
|
+
MAX_CONCURRENT: 10,
|
|
111
|
+
BATCH_SIZE: 10,
|
|
112
|
+
MAX_COMMENT_BUDGET: 1000,
|
|
113
|
+
MAX_COMMENTS_PER_POST: 200,
|
|
114
|
+
MIN_POSTS: 2,
|
|
115
|
+
MAX_POSTS: 50,
|
|
116
|
+
RETRY_COUNT: 5,
|
|
117
|
+
RETRY_DELAYS: [2000, 4000, 8000, 16000, 32000],
|
|
118
|
+
};
|
|
119
|
+
// ============================================================================
|
|
120
|
+
// CTR Weights for URL Ranking (inspired from CTR research)
|
|
121
|
+
// ============================================================================
|
|
122
|
+
export const CTR_WEIGHTS = {
|
|
123
|
+
1: 100.00,
|
|
124
|
+
2: 60.00,
|
|
125
|
+
3: 48.89,
|
|
126
|
+
4: 33.33,
|
|
127
|
+
5: 28.89,
|
|
128
|
+
6: 26.44,
|
|
129
|
+
7: 24.44,
|
|
130
|
+
8: 17.78,
|
|
131
|
+
9: 13.33,
|
|
132
|
+
10: 12.56,
|
|
133
|
+
};
|
|
134
|
+
// ============================================================================
|
|
135
|
+
// LLM Extraction Model (uses OPENROUTER for scrape_links AI extraction)
|
|
136
|
+
// ============================================================================
|
|
137
|
+
export const LLM_EXTRACTION = {
|
|
138
|
+
MODEL: process.env.LLM_EXTRACTION_MODEL || 'openai/gpt-oss-120b:nitro',
|
|
139
|
+
MAX_TOKENS: 8000,
|
|
140
|
+
ENABLE_REASONING: process.env.LLM_ENABLE_REASONING !== 'false', // Default true, can be disabled with 'false'
|
|
141
|
+
};
|
|
142
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/config/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,mBAAmB,EAAE,MAAM,eAAe,CAAC;AAE3E,iGAAiG;AAEjG,+EAA+E;AAC/E,8BAA8B;AAC9B,+EAA+E;AAE/E;;;;;;;GAOG;AACH,SAAS,YAAY,CACnB,KAAyB,EACzB,UAAkB,EAClB,GAAW,EACX,GAAW;IAEX,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IAEnC,IAAI,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;QAClB,OAAO,CAAC,IAAI,CAAC,4BAA4B,KAAK,oBAAoB,UAAU,EAAE,CAAC,CAAC;QAChF,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,IAAI,MAAM,GAAG,GAAG,EAAE,CAAC;QACjB,OAAO,CAAC,IAAI,CAAC,kBAAkB,MAAM,kBAAkB,GAAG,iBAAiB,GAAG,EAAE,CAAC,CAAC;QAClF,OAAO,GAAG,CAAC;IACb,CAAC;IAED,IAAI,MAAM,GAAG,GAAG,EAAE,CAAC;QACjB,OAAO,CAAC,IAAI,CAAC,kBAAkB,MAAM,kBAAkB,GAAG,iBAAiB,GAAG,EAAE,CAAC,CAAC;QAClF,OAAO,GAAG,CAAC;IACb,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAaD,IAAI,SAAS,GAAqB,IAAI,CAAC;AAEvC,MAAM,UAAU,QAAQ;IACtB,IAAI,SAAS;QAAE,OAAO,SAAS,CAAC;IAChC,SAAS,GAAG;QACV,eAAe,EAAE,OAAO,CAAC,GAAG,CAAC,gBAAgB,IAAI,EAAE;QACnD,cAAc,EAAE,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,SAAS;QACvD,gBAAgB,EAAE,OAAO,CAAC,GAAG,CAAC,gBAAgB,IAAI,SAAS;QAC3D,oBAAoB,EAAE,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,SAAS;KACpE,CAAC;IACF,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,+EAA+E;AAC/E,6BAA6B;AAC7B,+EAA+E;AAE/E,MAAM,CAAC,MAAM,QAAQ,GAAG;IACtB,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,8BAA8B;IAC3E,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,kBAAkB;IACvD,cAAc,EAAE,OAAO,CAAC,GAAG,CAAC,uBAAuB,IAAI,yBAAyB;IAChF,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,EAAE;IAC7C,0CAA0C;IAC1C,UAAU,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,CAAC;IAC5E,gBAAgB,EAAG,OAAO,CAAC,GAAG,CAAC,wBAAsD,IAAI,MAAM;IAC/F,2DAA2D;IAC3D,QAAQ,EAAE,YAAY,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,CAAC;CAC1D,CAAC;AAEX,+EAA+E;AAC/E,2BAA2B;AAC3B,+EAA+E;AAE/E,qEAAqE;AACrE,sDAAsD;AACtD,MAAM,CAAC,MAAM,MAAM,GAAG;IACpB,IAAI,EAAE,YAAY;IAClB,OAAO,EAAE,OAAO;IAChB,WAAW,EAAE,mBAAmB;CACxB,CAAC;AAcX,MAAM,UAAU,eAAe;IAC7B,MAAM,GAAG,GAAG,QAAQ,EAAE,CAAC;IACvB,OAAO;QACL,MAAM,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,gBAAgB,IAAI,GAAG,CAAC,oBAAoB,CAAC;QAC5D,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,cAAc;QAC5B,QAAQ,EAAE,CAAC,CAAC,GAAG,CAAC,eAAe;QAC/B,YAAY,EAAE,CAAC,CAAC,QAAQ,CAAC,OAAO;QAChC,aAAa,EAAE,CAAC,CAAC,QAAQ,CAAC,OAAO,EAAE,uCAAuC;KAC3E,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,UAA8B;IACjE,MAAM,QAAQ,GAAuC;QACnD,MAAM,EAAE,qLAAqL;QAC7L,MAAM,EAAE,gKAAgK;QACxK,QAAQ,EAAE,mJAAmJ;QAC7J,YAAY,EAAE,gJAAgJ;QAC9J,aAAa,EAAE,4KAA4K;KAC5L,CAAC;IACF,OAAO,QAAQ,CAAC,UAAU,CAAC,CAAC;AAC9B,CAAC;AAED,+EAA+E;AAC/E,mDAAmD;AACnD,+EAA+E;AAE/E,MAAM,CAAC,MAAM,OAAO,GAAG;IACrB,cAAc,EAAE,EAAE;IAClB,UAAU,EAAE,EAAE;IACd,iBAAiB,EAAE,KAAK;IACxB,QAAQ,EAAE,CAAC;IACX,QAAQ,EAAE,EAAE;IACZ,WAAW,EAAE,CAAC;IACd,YAAY,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAU;IACzC,iBAAiB,EAAE,sMAAsM;CACjN,CAAC;AAEX,+EAA+E;AAC/E,uBAAuB;AACvB,+EAA+E;AAE/E,MAAM,CAAC,MAAM,MAAM,GAAG;IACpB,cAAc,EAAE,EAAE;IAClB,UAAU,EAAE,EAAE;IACd,kBAAkB,EAAE,IAAI;IACxB,qBAAqB,EAAE,GAAG;IAC1B,SAAS,EAAE,CAAC;IACZ,SAAS,EAAE,EAAE;IACb,WAAW,EAAE,CAAC;IACd,YAAY,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,CAAU;CAC/C,CAAC;AAEX,+EAA+E;AAC/E,2DAA2D;AAC3D,+EAA+E;AAE/E,MAAM,CAAC,MAAM,WAAW,GAA2B;IACjD,CAAC,EAAE,MAAM;IACT,CAAC,EAAE,KAAK;IACR,CAAC,EAAE,KAAK;IACR,CAAC,EAAE,KAAK;IACR,CAAC,EAAE,KAAK;IACR,CAAC,EAAE,KAAK;IACR,CAAC,EAAE,KAAK;IACR,CAAC,EAAE,KAAK;IACR,CAAC,EAAE,KAAK;IACR,EAAE,EAAE,KAAK;CACD,CAAC;AAEX,+EAA+E;AAC/E,wEAAwE;AACxE,+EAA+E;AAE/E,MAAM,CAAC,MAAM,cAAc,GAAG;IAC5B,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,2BAA2B;IACtE,UAAU,EAAE,IAAI;IAChB,gBAAgB,EAAE,OAAO,CAAC,GAAG,CAAC,oBAAoB,KAAK,OAAO,EAAE,6CAA6C;CACrG,CAAC"}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* YAML Configuration Loader
|
|
3
|
+
* Loads tools.yaml and generates MCP-compatible tool definitions
|
|
4
|
+
*/
|
|
5
|
+
import { z } from 'zod';
|
|
6
|
+
import type { YamlConfig, YamlToolConfig, YamlParameter, McpTool, LoadedTool } from './types.js';
|
|
7
|
+
/**
|
|
8
|
+
* Recursively convert YAML parameter to Zod schema
|
|
9
|
+
*/
|
|
10
|
+
declare function yamlParamToZod(param: YamlParameter): z.ZodTypeAny;
|
|
11
|
+
/**
|
|
12
|
+
* Convert YAML parameters to Zod object schema
|
|
13
|
+
*/
|
|
14
|
+
declare function yamlParamsToZodSchema(params: Record<string, YamlParameter>): z.ZodObject<Record<string, z.ZodTypeAny>>;
|
|
15
|
+
/**
|
|
16
|
+
* Load and parse tools.yaml
|
|
17
|
+
*/
|
|
18
|
+
export declare function loadYamlConfig(): YamlConfig;
|
|
19
|
+
/**
|
|
20
|
+
* Load all tools from YAML configuration
|
|
21
|
+
*/
|
|
22
|
+
export declare function loadToolsFromYaml(): LoadedTool[];
|
|
23
|
+
/**
|
|
24
|
+
* Get tool configuration by name
|
|
25
|
+
*/
|
|
26
|
+
export declare function getToolConfig(name: string): YamlToolConfig | undefined;
|
|
27
|
+
/**
|
|
28
|
+
* Get schema descriptions for a tool (for injecting into existing Zod schemas)
|
|
29
|
+
*/
|
|
30
|
+
export declare function getSchemaDescriptions(name: string): Record<string, string> | undefined;
|
|
31
|
+
/**
|
|
32
|
+
* Get Zod schema by reference name
|
|
33
|
+
*/
|
|
34
|
+
export declare function getZodSchemaByRef(ref: string): z.ZodTypeAny | undefined;
|
|
35
|
+
/**
|
|
36
|
+
* Generate complete MCP tools list with all schemas resolved
|
|
37
|
+
*/
|
|
38
|
+
export declare function generateMcpTools(): McpTool[];
|
|
39
|
+
export { yamlParamsToZodSchema, yamlParamToZod };
|
|
40
|
+
//# sourceMappingURL=loader.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"loader.d.ts","sourceRoot":"","sources":["../../src/config/loader.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAMH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAGxB,OAAO,KAAK,EACV,UAAU,EACV,cAAc,EACd,aAAa,EAEb,OAAO,EACP,UAAU,EACX,MAAM,YAAY,CAAC;AA8DpB;;GAEG;AACH,iBAAS,cAAc,CAAC,KAAK,EAAE,aAAa,GAAG,CAAC,CAAC,UAAU,CAqE1D;AAED;;GAEG;AACH,iBAAS,qBAAqB,CAC5B,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,GACpC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC,CAU3C;AAyBD;;GAEG;AACH,wBAAgB,cAAc,IAAI,UAAU,CAM3C;AAwCD;;GAEG;AACH,wBAAgB,iBAAiB,IAAI,UAAU,EAAE,CAYhD;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,cAAc,GAAG,SAAS,CAGtE;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAGtF;AAmBD;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,CAAC,CAAC,UAAU,GAAG,SAAS,CAEvE;AAED;;GAEG;AACH,wBAAgB,gBAAgB,IAAI,OAAO,EAAE,CA+B5C;AAMD,OAAO,EAAE,qBAAqB,EAAE,cAAc,EAAE,CAAC"}
|