@j0hanz/fetch-url-mcp 1.9.1 → 1.9.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. package/dist/http/auth.d.ts +0 -1
  2. package/dist/http/auth.d.ts.map +1 -1
  3. package/dist/http/auth.js +1 -13
  4. package/dist/http/native.d.ts.map +1 -1
  5. package/dist/http/native.js +2 -5
  6. package/dist/lib/content.d.ts.map +1 -1
  7. package/dist/lib/content.js +301 -350
  8. package/dist/lib/core.d.ts +78 -71
  9. package/dist/lib/core.d.ts.map +1 -1
  10. package/dist/lib/core.js +308 -372
  11. package/dist/lib/fetch-pipeline.d.ts +2 -6
  12. package/dist/lib/fetch-pipeline.d.ts.map +1 -1
  13. package/dist/lib/fetch-pipeline.js +51 -137
  14. package/dist/lib/http.d.ts.map +1 -1
  15. package/dist/lib/http.js +188 -130
  16. package/dist/lib/mcp-tools.d.ts +3 -5
  17. package/dist/lib/mcp-tools.d.ts.map +1 -1
  18. package/dist/lib/mcp-tools.js +22 -58
  19. package/dist/lib/task-handlers.js +4 -4
  20. package/dist/lib/utils.d.ts +6 -0
  21. package/dist/lib/utils.d.ts.map +1 -1
  22. package/dist/lib/utils.js +23 -0
  23. package/dist/resources/index.js +1 -1
  24. package/dist/schemas.d.ts +0 -1
  25. package/dist/schemas.d.ts.map +1 -1
  26. package/dist/schemas.js +4 -6
  27. package/dist/server.js +1 -1
  28. package/dist/tasks/owner.d.ts +1 -1
  29. package/dist/tasks/owner.d.ts.map +1 -1
  30. package/dist/tasks/tool-registry.d.ts +1 -1
  31. package/dist/tasks/tool-registry.d.ts.map +1 -1
  32. package/dist/tools/fetch-url.d.ts +2 -3
  33. package/dist/tools/fetch-url.d.ts.map +1 -1
  34. package/dist/tools/fetch-url.js +89 -152
  35. package/dist/transform/transform.d.ts +8 -0
  36. package/dist/transform/transform.d.ts.map +1 -1
  37. package/dist/transform/transform.js +109 -108
  38. package/dist/transform/worker-pool.d.ts +3 -6
  39. package/dist/transform/worker-pool.d.ts.map +1 -1
  40. package/dist/transform/worker-pool.js +148 -118
  41. package/package.json +2 -1
package/dist/lib/core.js CHANGED
@@ -56,129 +56,117 @@ function loadEnvFileIfAvailable() {
56
56
  }
57
57
  loadEnvFileIfAvailable();
58
58
  const { env } = process;
59
- function formatHostForUrl(hostname) {
60
- if (hostname.includes(':') && !hostname.startsWith('['))
61
- return `[${hostname}]`;
62
- return hostname;
63
- }
64
- function normalizeHostValue(value) {
65
- const raw = value.trim();
66
- if (!raw)
67
- return null;
68
- if (raw.includes('://') && URL.canParse(raw)) {
69
- return normalizeHostname(new URL(raw).hostname);
70
- }
71
- const candidateUrl = `http://${raw}`;
72
- if (URL.canParse(candidateUrl)) {
73
- return normalizeHostname(new URL(candidateUrl).hostname);
74
- }
75
- const lowered = raw.toLowerCase();
76
- if (lowered.startsWith('[')) {
77
- const end = lowered.indexOf(']');
78
- if (end === -1)
59
+ const EnvParser = {
60
+ integerValue(envValue, min, max) {
61
+ if (!envValue)
79
62
  return null;
80
- return normalizeHostname(lowered.slice(1, end));
81
- }
82
- if (isIP(lowered) === 6)
83
- return stripTrailingDots(lowered);
84
- const firstColon = lowered.indexOf(':');
85
- if (firstColon === -1)
86
- return normalizeHostname(lowered);
87
- if (lowered.includes(':', firstColon + 1))
88
- return null;
89
- const host = lowered.slice(0, firstColon);
90
- return host ? normalizeHostname(host) : null;
91
- }
92
- function parseIntegerValue(envValue, min, max) {
93
- if (!envValue)
94
- return null;
95
- const parsed = Number.parseInt(envValue, 10);
96
- if (Number.isNaN(parsed))
97
- return null;
98
- if (min !== undefined && parsed < min)
99
- return null;
100
- if (max !== undefined && parsed > max)
101
- return null;
102
- return parsed;
103
- }
104
- function parseOptionalInteger(envValue, min, max) {
105
- return parseIntegerValue(envValue, min, max) ?? undefined;
106
- }
107
- function parseInteger(envValue, defaultValue, min, max) {
108
- return parseIntegerValue(envValue, min, max) ?? defaultValue;
109
- }
110
- function parseBoolean(envValue, defaultValue) {
111
- if (!envValue)
112
- return defaultValue;
113
- return envValue.trim().toLowerCase() !== 'false';
114
- }
115
- function parseList(envValue) {
116
- if (!envValue)
117
- return [];
118
- return envValue
119
- .split(/[\s,]+/)
120
- .map((entry) => entry.trim())
121
- .filter(Boolean);
122
- }
123
- function parseListOrDefault(envValue, defaultValue) {
124
- const parsed = parseList(envValue);
125
- return parsed.length > 0 ? parsed : [...defaultValue];
126
- }
127
- function normalizeLocale(value) {
128
- if (!value)
129
- return undefined;
130
- const trimmed = value.trim();
131
- if (!trimmed)
132
- return undefined;
133
- const lowered = trimmed.toLowerCase();
134
- if (lowered === 'system' || lowered === 'default')
135
- return undefined;
136
- return trimmed;
137
- }
138
- function isLogLevel(value) {
139
- return ALLOWED_LOG_LEVELS.has(value);
140
- }
141
- function parseLogLevel(envValue) {
142
- if (!envValue)
143
- return 'info';
144
- const level = envValue.toLowerCase();
145
- return isLogLevel(level) ? level : 'info';
146
- }
147
- function parseTransformWorkerMode(envValue) {
148
- if (!envValue)
63
+ const parsed = Number.parseInt(envValue, 10);
64
+ if (Number.isNaN(parsed))
65
+ return null;
66
+ if (min !== undefined && parsed < min)
67
+ return null;
68
+ if (max !== undefined && parsed > max)
69
+ return null;
70
+ return parsed;
71
+ },
72
+ optionalInteger(envValue, min, max) {
73
+ return EnvParser.integerValue(envValue, min, max) ?? undefined;
74
+ },
75
+ integer(envValue, defaultValue, min, max) {
76
+ return EnvParser.integerValue(envValue, min, max) ?? defaultValue;
77
+ },
78
+ boolean(envValue, defaultValue) {
79
+ if (!envValue)
80
+ return defaultValue;
81
+ return envValue.trim().toLowerCase() !== 'false';
82
+ },
83
+ list(envValue, defaultValue) {
84
+ if (!envValue)
85
+ return defaultValue ? [...defaultValue] : [];
86
+ const parsed = envValue
87
+ .split(/[\s,]+/)
88
+ .map((entry) => entry.trim())
89
+ .filter(Boolean);
90
+ return parsed.length > 0 || !defaultValue ? parsed : [...defaultValue];
91
+ },
92
+ locale(value) {
93
+ if (!value)
94
+ return undefined;
95
+ const trimmed = value.trim();
96
+ if (!trimmed)
97
+ return undefined;
98
+ const lowered = trimmed.toLowerCase();
99
+ if (lowered === 'system' || lowered === 'default')
100
+ return undefined;
101
+ return trimmed;
102
+ },
103
+ logLevel(envValue) {
104
+ if (!envValue)
105
+ return 'info';
106
+ const level = envValue.toLowerCase();
107
+ return ALLOWED_LOG_LEVELS.has(level) ? level : 'info';
108
+ },
109
+ transformWorkerMode(envValue) {
110
+ if (!envValue)
111
+ return 'threads';
112
+ const normalized = envValue.trim().toLowerCase();
113
+ if (normalized === 'process' || normalized === 'fork')
114
+ return 'process';
149
115
  return 'threads';
150
- const normalized = envValue.trim().toLowerCase();
151
- if (normalized === 'process' || normalized === 'fork')
152
- return 'process';
153
- return 'threads';
154
- }
155
- function parsePort(envValue) {
156
- if (envValue?.trim() === '0')
157
- return 0;
158
- return parseInteger(envValue, 3000, 1024, 65535);
159
- }
160
- function parseUrlEnv(value, name) {
161
- if (!value)
162
- return undefined;
163
- if (!URL.canParse(value)) {
164
- throw new ConfigError(`Invalid ${name} value: ${value}`);
165
- }
166
- return new URL(value);
167
- }
168
- function readUrlEnv(name) {
169
- return parseUrlEnv(env[name], name);
170
- }
171
- function parseAllowedHosts(envValue) {
172
- return new Set(parseList(envValue)
173
- .map(normalizeHostValue)
174
- .filter((h) => h !== null));
175
- }
176
- function readOptionalFilePath(value) {
177
- if (!value)
178
- return undefined;
179
- const trimmed = value.trim();
180
- return trimmed.length > 0 ? trimmed : undefined;
181
- }
116
+ },
117
+ url(value, name) {
118
+ if (!value)
119
+ return undefined;
120
+ if (!URL.canParse(value)) {
121
+ throw new ConfigError(`Invalid ${name} value: ${value}`);
122
+ }
123
+ return new URL(value);
124
+ },
125
+ allowedHosts(envValue) {
126
+ return new Set(EnvParser.list(envValue)
127
+ .map((h) => EnvParser.normalizeHostValue(h))
128
+ .filter((h) => h !== null));
129
+ },
130
+ optionalFilePath(value) {
131
+ if (!value)
132
+ return undefined;
133
+ const trimmed = value.trim();
134
+ return trimmed.length > 0 ? trimmed : undefined;
135
+ },
136
+ normalizeHostValue(value) {
137
+ const raw = value.trim();
138
+ if (!raw)
139
+ return null;
140
+ if (raw.includes('://') && URL.canParse(raw)) {
141
+ return normalizeHostname(new URL(raw).hostname);
142
+ }
143
+ const candidateUrl = `http://${raw}`;
144
+ if (URL.canParse(candidateUrl)) {
145
+ return normalizeHostname(new URL(candidateUrl).hostname);
146
+ }
147
+ const lowered = raw.toLowerCase();
148
+ if (lowered.startsWith('[')) {
149
+ const end = lowered.indexOf(']');
150
+ if (end === -1)
151
+ return null;
152
+ return normalizeHostname(lowered.slice(1, end));
153
+ }
154
+ if (isIP(lowered) === 6)
155
+ return stripTrailingDots(lowered);
156
+ const firstColon = lowered.indexOf(':');
157
+ if (firstColon === -1)
158
+ return normalizeHostname(lowered);
159
+ if (lowered.includes(':', firstColon + 1))
160
+ return null;
161
+ const host = lowered.slice(0, firstColon);
162
+ return host ? normalizeHostname(host) : null;
163
+ },
164
+ formatHostForUrl(hostname) {
165
+ if (hostname.includes(':') && !hostname.startsWith('['))
166
+ return `[${hostname}]`;
167
+ return hostname;
168
+ },
169
+ };
182
170
  function assertFileReadable(filePath, envVar) {
183
171
  try {
184
172
  accessSync(filePath, fsConstants.R_OK);
@@ -188,55 +176,59 @@ function assertFileReadable(filePath, envVar) {
188
176
  }
189
177
  }
190
178
  const MAX_HTML_BYTES = 10 * 1024 * 1024;
191
- const MAX_INLINE_CONTENT_CHARS = parseInteger(env['MAX_INLINE_CONTENT_CHARS'], 0, 0, MAX_HTML_BYTES);
179
+ const MAX_INLINE_CONTENT_CHARS = EnvParser.integer(env['MAX_INLINE_CONTENT_CHARS'], 0, 0, MAX_HTML_BYTES);
192
180
  const DEFAULT_SESSION_TTL_MS = 30 * 60 * 1000;
193
181
  const DEFAULT_SESSION_INIT_TIMEOUT_MS = 10000;
194
182
  const DEFAULT_MAX_SESSIONS = 200;
195
183
  const DEFAULT_USER_AGENT = `fetch-url-mcp/${serverVersion}`;
196
184
  const DEFAULT_TOOL_TIMEOUT_PADDING_MS = 5000;
197
185
  const DEFAULT_TRANSFORM_TIMEOUT_MS = 30000;
198
- const DEFAULT_FETCH_TIMEOUT_MS = parseInteger(env['FETCH_TIMEOUT_MS'], 15000, 1000, 60000);
186
+ const DEFAULT_FETCH_TIMEOUT_MS = EnvParser.integer(env['FETCH_TIMEOUT_MS'], 15000, 1000, 60000);
199
187
  const DEFAULT_TOOL_TIMEOUT_MS = DEFAULT_FETCH_TIMEOUT_MS +
200
188
  DEFAULT_TRANSFORM_TIMEOUT_MS +
201
189
  DEFAULT_TOOL_TIMEOUT_PADDING_MS;
202
- const DEFAULT_TASKS_MAX_TOTAL = parseInteger(env['TASKS_MAX_TOTAL'], 5000, 1);
203
- const DEFAULT_TASKS_MAX_PER_OWNER = parseInteger(env['TASKS_MAX_PER_OWNER'], 1000, 1);
190
+ const DEFAULT_TASKS_MAX_TOTAL = EnvParser.integer(env['TASKS_MAX_TOTAL'], 5000, 1);
191
+ const DEFAULT_TASKS_MAX_PER_OWNER = EnvParser.integer(env['TASKS_MAX_PER_OWNER'], 1000, 1);
204
192
  const RESOLVED_TASKS_MAX_PER_OWNER = Math.min(DEFAULT_TASKS_MAX_PER_OWNER, DEFAULT_TASKS_MAX_TOTAL);
205
193
  function resolveWorkerResourceLimits() {
206
- const limits = {};
207
- let hasAny = false;
208
- const entries = [
209
- [
210
- 'maxOldGenerationSizeMb',
211
- parseOptionalInteger(env['TRANSFORM_WORKER_MAX_OLD_GENERATION_MB'], 1),
212
- ],
213
- [
214
- 'maxYoungGenerationSizeMb',
215
- parseOptionalInteger(env['TRANSFORM_WORKER_MAX_YOUNG_GENERATION_MB'], 1),
216
- ],
217
- [
218
- 'codeRangeSizeMb',
219
- parseOptionalInteger(env['TRANSFORM_WORKER_CODE_RANGE_MB'], 1),
220
- ],
221
- ['stackSizeMb', parseOptionalInteger(env['TRANSFORM_WORKER_STACK_MB'], 1)],
222
- ];
223
- for (const [key, value] of entries) {
224
- if (value === undefined)
225
- continue;
226
- limits[key] = value;
227
- hasAny = true;
194
+ const maxOldGenerationSizeMb = EnvParser.optionalInteger(env['TRANSFORM_WORKER_MAX_OLD_GENERATION_MB'], 1);
195
+ const maxYoungGenerationSizeMb = EnvParser.optionalInteger(env['TRANSFORM_WORKER_MAX_YOUNG_GENERATION_MB'], 1);
196
+ const codeRangeSizeMb = EnvParser.optionalInteger(env['TRANSFORM_WORKER_CODE_RANGE_MB'], 1);
197
+ const stackSizeMb = EnvParser.optionalInteger(env['TRANSFORM_WORKER_STACK_MB'], 1);
198
+ if (maxOldGenerationSizeMb === undefined &&
199
+ maxYoungGenerationSizeMb === undefined &&
200
+ codeRangeSizeMb === undefined &&
201
+ stackSizeMb === undefined) {
202
+ return undefined;
228
203
  }
229
- return hasAny ? limits : undefined;
230
- }
231
- function readOAuthUrls(baseUrl) {
232
- const issuerUrl = readUrlEnv('OAUTH_ISSUER_URL');
233
- const authorizationUrl = readUrlEnv('OAUTH_AUTHORIZATION_URL');
234
- const tokenUrl = readUrlEnv('OAUTH_TOKEN_URL');
235
- const revocationUrl = readUrlEnv('OAUTH_REVOCATION_URL');
236
- const registrationUrl = readUrlEnv('OAUTH_REGISTRATION_URL');
237
- const introspectionUrl = readUrlEnv('OAUTH_INTROSPECTION_URL');
204
+ const limits = {};
205
+ if (maxOldGenerationSizeMb !== undefined)
206
+ limits.maxOldGenerationSizeMb = maxOldGenerationSizeMb;
207
+ if (maxYoungGenerationSizeMb !== undefined)
208
+ limits.maxYoungGenerationSizeMb = maxYoungGenerationSizeMb;
209
+ if (codeRangeSizeMb !== undefined)
210
+ limits.codeRangeSizeMb = codeRangeSizeMb;
211
+ if (stackSizeMb !== undefined)
212
+ limits.stackSizeMb = stackSizeMb;
213
+ return limits;
214
+ }
215
+ function buildAuthConfig(baseUrl) {
216
+ const issuerUrl = EnvParser.url(env['OAUTH_ISSUER_URL'], 'OAUTH_ISSUER_URL');
217
+ const authorizationUrl = EnvParser.url(env['OAUTH_AUTHORIZATION_URL'], 'OAUTH_AUTHORIZATION_URL');
218
+ const tokenUrl = EnvParser.url(env['OAUTH_TOKEN_URL'], 'OAUTH_TOKEN_URL');
219
+ const revocationUrl = EnvParser.url(env['OAUTH_REVOCATION_URL'], 'OAUTH_REVOCATION_URL');
220
+ const registrationUrl = EnvParser.url(env['OAUTH_REGISTRATION_URL'], 'OAUTH_REGISTRATION_URL');
221
+ const introspectionUrl = EnvParser.url(env['OAUTH_INTROSPECTION_URL'], 'OAUTH_INTROSPECTION_URL');
238
222
  const resourceUrl = new URL('/mcp', baseUrl);
223
+ const oauthConfigured = issuerUrl !== undefined ||
224
+ authorizationUrl !== undefined ||
225
+ tokenUrl !== undefined ||
226
+ introspectionUrl !== undefined;
227
+ const tokens = EnvParser.list(env['ACCESS_TOKENS']);
228
+ if (env['API_KEY'])
229
+ tokens.push(env['API_KEY']);
239
230
  return {
231
+ mode: oauthConfigured ? 'oauth' : 'static',
240
232
  issuerUrl,
241
233
  authorizationUrl,
242
234
  tokenUrl,
@@ -244,40 +236,17 @@ function readOAuthUrls(baseUrl) {
244
236
  registrationUrl,
245
237
  introspectionUrl,
246
238
  resourceUrl,
247
- };
248
- }
249
- function resolveAuthMode(urls) {
250
- const oauthConfigured = [
251
- urls.issuerUrl,
252
- urls.authorizationUrl,
253
- urls.tokenUrl,
254
- urls.introspectionUrl,
255
- ].some((value) => value !== undefined);
256
- return oauthConfigured ? 'oauth' : 'static';
257
- }
258
- function collectStaticTokens() {
259
- const tokens = parseList(env['ACCESS_TOKENS']);
260
- if (env['API_KEY'])
261
- tokens.push(env['API_KEY']);
262
- return [...new Set(tokens)];
263
- }
264
- function buildAuthConfig(baseUrl) {
265
- const urls = readOAuthUrls(baseUrl);
266
- const mode = resolveAuthMode(urls);
267
- return {
268
- mode,
269
- ...urls,
270
- requiredScopes: parseList(env['OAUTH_REQUIRED_SCOPES']),
239
+ requiredScopes: EnvParser.list(env['OAUTH_REQUIRED_SCOPES']),
271
240
  clientId: env['OAUTH_CLIENT_ID'],
272
241
  clientSecret: env['OAUTH_CLIENT_SECRET'],
273
242
  introspectionTimeoutMs: 5000,
274
- staticTokens: collectStaticTokens(),
243
+ staticTokens: [...new Set(tokens)],
275
244
  };
276
245
  }
277
246
  function buildHttpsConfig() {
278
- const keyFile = readOptionalFilePath(env['SERVER_TLS_KEY_FILE']);
279
- const certFile = readOptionalFilePath(env['SERVER_TLS_CERT_FILE']);
280
- const caFile = readOptionalFilePath(env['SERVER_TLS_CA_FILE']);
247
+ const keyFile = EnvParser.optionalFilePath(env['SERVER_TLS_KEY_FILE']);
248
+ const certFile = EnvParser.optionalFilePath(env['SERVER_TLS_CERT_FILE']);
249
+ const caFile = EnvParser.optionalFilePath(env['SERVER_TLS_CA_FILE']);
281
250
  if (keyFile)
282
251
  assertFileReadable(keyFile, 'SERVER_TLS_KEY_FILE');
283
252
  if (certFile)
@@ -310,23 +279,17 @@ const BLOCKED_HOSTS = new Set([
310
279
  'instance-data',
311
280
  ]);
312
281
  const host = (env['HOST'] ?? LOOPBACK_V4).trim();
313
- const port = parsePort(env['PORT']);
282
+ const port = env['PORT']?.trim() === '0'
283
+ ? 0
284
+ : EnvParser.integer(env['PORT'], 3000, 1024, 65535);
314
285
  const httpsConfig = buildHttpsConfig();
315
- const maxConnections = parseInteger(env['SERVER_MAX_CONNECTIONS'], 0, 0);
316
- const headersTimeoutMs = parseOptionalInteger(env['SERVER_HEADERS_TIMEOUT_MS'], 1);
317
- const requestTimeoutMs = parseOptionalInteger(env['SERVER_REQUEST_TIMEOUT_MS'], 0);
318
- const keepAliveTimeoutMs = parseOptionalInteger(env['SERVER_KEEP_ALIVE_TIMEOUT_MS'], 1);
319
- const keepAliveTimeoutBufferMs = parseOptionalInteger(env['SERVER_KEEP_ALIVE_TIMEOUT_BUFFER_MS'], 0);
320
- const maxHeadersCount = parseOptionalInteger(env['SERVER_MAX_HEADERS_COUNT'], 1);
321
- const blockPrivateConnections = parseBoolean(env['SERVER_BLOCK_PRIVATE_CONNECTIONS'], false);
322
- const allowRemote = parseBoolean(env['ALLOW_REMOTE'], false);
323
- const requireProtocolVersionHeaderOnSessionInit = parseBoolean(env['MCP_STRICT_PROTOCOL_VERSION_HEADER'], true);
324
- const baseUrl = new URL(`${httpsConfig.enabled ? 'https' : 'http'}://${formatHostForUrl(host)}:${port}`);
286
+ const allowRemote = EnvParser.boolean(env['ALLOW_REMOTE'], false);
287
+ const baseUrl = new URL(`${httpsConfig.enabled ? 'https' : 'http'}://${EnvParser.formatHostForUrl(host)}:${port}`);
325
288
  const runtimeState = {
326
289
  httpMode: false,
327
290
  };
328
- export const config = {
329
- server: {
291
+ function buildServerConfig() {
292
+ return {
330
293
  name: 'fetch-url-mcp',
331
294
  version: serverVersion,
332
295
  port,
@@ -336,56 +299,57 @@ export const config = {
336
299
  sessionInitTimeoutMs: DEFAULT_SESSION_INIT_TIMEOUT_MS,
337
300
  maxSessions: DEFAULT_MAX_SESSIONS,
338
301
  http: {
339
- headersTimeoutMs,
340
- requestTimeoutMs,
341
- keepAliveTimeoutMs,
342
- keepAliveTimeoutBufferMs,
343
- maxHeadersCount,
344
- maxConnections,
345
- blockPrivateConnections,
346
- requireProtocolVersionHeaderOnSessionInit,
302
+ headersTimeoutMs: EnvParser.optionalInteger(env['SERVER_HEADERS_TIMEOUT_MS'], 1),
303
+ requestTimeoutMs: EnvParser.optionalInteger(env['SERVER_REQUEST_TIMEOUT_MS'], 0),
304
+ keepAliveTimeoutMs: EnvParser.optionalInteger(env['SERVER_KEEP_ALIVE_TIMEOUT_MS'], 1),
305
+ keepAliveTimeoutBufferMs: EnvParser.optionalInteger(env['SERVER_KEEP_ALIVE_TIMEOUT_BUFFER_MS'], 0),
306
+ maxHeadersCount: EnvParser.optionalInteger(env['SERVER_MAX_HEADERS_COUNT'], 1),
307
+ maxConnections: EnvParser.integer(env['SERVER_MAX_CONNECTIONS'], 0, 0),
308
+ blockPrivateConnections: EnvParser.boolean(env['SERVER_BLOCK_PRIVATE_CONNECTIONS'], false),
347
309
  shutdownCloseIdleConnections: true,
348
310
  shutdownCloseAllConnections: false,
349
311
  },
350
- },
351
- fetcher: {
312
+ };
313
+ }
314
+ function buildFetcherConfig() {
315
+ return {
352
316
  timeout: DEFAULT_FETCH_TIMEOUT_MS,
353
317
  maxRedirects: 5,
354
318
  userAgent: env['USER_AGENT'] ?? DEFAULT_USER_AGENT,
355
319
  maxContentLength: MAX_HTML_BYTES,
356
- },
357
- transform: {
320
+ };
321
+ }
322
+ function buildTransformConfig() {
323
+ return {
358
324
  timeoutMs: DEFAULT_TRANSFORM_TIMEOUT_MS,
359
325
  stageWarnRatio: 0.5,
360
326
  metadataFormat: 'markdown',
361
327
  maxWorkerScale: 4,
362
- cancelAckTimeoutMs: parseInteger(env['TRANSFORM_CANCEL_ACK_TIMEOUT_MS'], 200, 50, 5000),
363
- workerMode: parseTransformWorkerMode(env['TRANSFORM_WORKER_MODE']),
328
+ cancelAckTimeoutMs: EnvParser.integer(env['TRANSFORM_CANCEL_ACK_TIMEOUT_MS'], 200, 50, 5000),
329
+ workerMode: EnvParser.transformWorkerMode(env['TRANSFORM_WORKER_MODE']),
364
330
  workerResourceLimits: resolveWorkerResourceLimits(),
365
- },
366
- tools: {
367
- enabled: ['fetch-url'],
368
- timeoutMs: DEFAULT_TOOL_TIMEOUT_MS,
369
- },
370
- tasks: {
331
+ };
332
+ }
333
+ function buildTasksConfig() {
334
+ return {
371
335
  maxTotal: DEFAULT_TASKS_MAX_TOTAL,
372
336
  maxPerOwner: RESOLVED_TASKS_MAX_PER_OWNER,
373
- emitStatusNotifications: parseBoolean(env['TASKS_STATUS_NOTIFICATIONS'], false),
374
- requireInterception: parseBoolean(env['TASKS_REQUIRE_INTERCEPTION'], true),
375
- },
376
- cache: {
377
- enabled: parseBoolean(env['CACHE_ENABLED'], true),
337
+ emitStatusNotifications: EnvParser.boolean(env['TASKS_STATUS_NOTIFICATIONS'], false),
338
+ requireInterception: EnvParser.boolean(env['TASKS_REQUIRE_INTERCEPTION'], true),
339
+ };
340
+ }
341
+ function buildCacheConfig() {
342
+ return {
343
+ enabled: EnvParser.boolean(env['CACHE_ENABLED'], true),
378
344
  ttl: 86400,
379
345
  maxKeys: 100,
380
- maxSizeBytes: 50 * 1024 * 1024, // 50MB
381
- },
382
- extraction: {
383
- maxBlockLength: 5000,
384
- minParagraphLength: 10,
385
- },
386
- noiseRemoval: {
387
- extraTokens: parseList(env['FETCH_URL_MCP_EXTRA_NOISE_TOKENS']),
388
- extraSelectors: parseList(env['FETCH_URL_MCP_EXTRA_NOISE_SELECTORS']),
346
+ maxSizeBytes: 50 * 1024 * 1024,
347
+ };
348
+ }
349
+ function buildNoiseRemovalConfig() {
350
+ return {
351
+ extraTokens: EnvParser.list(env['FETCH_URL_MCP_EXTRA_NOISE_TOKENS']),
352
+ extraSelectors: EnvParser.list(env['FETCH_URL_MCP_EXTRA_NOISE_SELECTORS']),
389
353
  enabledCategories: [
390
354
  'cookie-banners',
391
355
  'newsletters',
@@ -402,19 +366,38 @@ export const config = {
402
366
  stickyFixed: 30,
403
367
  threshold: 50,
404
368
  },
405
- },
406
- markdownCleanup: {
369
+ };
370
+ }
371
+ function buildMarkdownCleanupConfig() {
372
+ return {
407
373
  promoteOrphanHeadings: true,
408
374
  removeSkipLinks: true,
409
375
  removeTocBlocks: true,
410
376
  removeTypeDocComments: true,
411
- headingKeywords: parseListOrDefault(env['MARKDOWN_HEADING_KEYWORDS'], DEFAULT_HEADING_KEYWORDS),
377
+ headingKeywords: EnvParser.list(env['MARKDOWN_HEADING_KEYWORDS'], DEFAULT_HEADING_KEYWORDS),
378
+ };
379
+ }
380
+ export const config = {
381
+ server: buildServerConfig(),
382
+ fetcher: buildFetcherConfig(),
383
+ transform: buildTransformConfig(),
384
+ tools: {
385
+ enabled: ['fetch-url'],
386
+ timeoutMs: DEFAULT_TOOL_TIMEOUT_MS,
412
387
  },
388
+ tasks: buildTasksConfig(),
389
+ cache: buildCacheConfig(),
390
+ extraction: {
391
+ maxBlockLength: 5000,
392
+ minParagraphLength: 10,
393
+ },
394
+ noiseRemoval: buildNoiseRemovalConfig(),
395
+ markdownCleanup: buildMarkdownCleanupConfig(),
413
396
  i18n: {
414
- locale: normalizeLocale(env['FETCH_URL_MCP_LOCALE']),
397
+ locale: EnvParser.locale(env['FETCH_URL_MCP_LOCALE']),
415
398
  },
416
399
  logging: {
417
- level: parseLogLevel(env['LOG_LEVEL']),
400
+ level: EnvParser.logLevel(env['LOG_LEVEL']),
418
401
  format: env['LOG_FORMAT']?.toLowerCase() === 'json' ? 'json' : 'text',
419
402
  },
420
403
  constants: {
@@ -424,10 +407,10 @@ export const config = {
424
407
  },
425
408
  security: {
426
409
  blockedHosts: BLOCKED_HOSTS,
427
- allowedHosts: parseAllowedHosts(env['ALLOWED_HOSTS']),
410
+ allowedHosts: EnvParser.allowedHosts(env['ALLOWED_HOSTS']),
428
411
  apiKey: env['API_KEY'],
429
412
  allowRemote,
430
- allowLocalFetch: parseBoolean(env['ALLOW_LOCAL_FETCH'], false),
413
+ allowLocalFetch: EnvParser.boolean(env['ALLOW_LOCAL_FETCH'], false),
431
414
  },
432
415
  auth: buildAuthConfig(baseUrl),
433
416
  rateLimit: {
@@ -441,40 +424,30 @@ export const config = {
441
424
  export function enableHttpMode() {
442
425
  runtimeState.httpMode = true;
443
426
  }
444
- const CACHE_CONSTANTS = {
445
- URL_HASH_LENGTH: 32,
446
- VARY_HASH_LENGTH: 16,
447
- };
448
- function createHashFragment(input, length) {
449
- return sha256Hex(input).substring(0, length);
450
- }
451
- function buildCacheKey(namespace, urlHash, varyHash) {
452
- return varyHash
453
- ? `${namespace}:${urlHash}.${varyHash}`
454
- : `${namespace}:${urlHash}`;
455
- }
456
- function resolveVaryString(vary) {
457
- if (typeof vary === 'string')
458
- return vary;
459
- try {
460
- return stableJsonStringify(vary);
461
- }
462
- catch {
463
- return null;
464
- }
465
- }
466
427
  export function createCacheKey(namespace, url, vary) {
467
428
  if (!namespace || !url)
468
429
  return null;
469
- const urlHash = createHashFragment(url, CACHE_CONSTANTS.URL_HASH_LENGTH);
430
+ const urlHash = sha256Hex(url).substring(0, 32);
470
431
  if (!vary)
471
- return buildCacheKey(namespace, urlHash);
472
- const varyString = resolveVaryString(vary);
432
+ return `${namespace}:${urlHash}`;
433
+ const varyString = typeof vary === 'string'
434
+ ? vary
435
+ : (() => {
436
+ try {
437
+ return stableJsonStringify(vary);
438
+ }
439
+ catch {
440
+ return null;
441
+ }
442
+ })();
473
443
  if (varyString === null)
474
444
  return null;
475
- return buildCacheKey(namespace, urlHash, varyString
476
- ? createHashFragment(varyString, CACHE_CONSTANTS.VARY_HASH_LENGTH)
477
- : undefined);
445
+ const varyHash = varyString
446
+ ? sha256Hex(varyString).substring(0, 16)
447
+ : undefined;
448
+ return varyHash
449
+ ? `${namespace}:${urlHash}.${varyHash}`
450
+ : `${namespace}:${urlHash}`;
478
451
  }
479
452
  export function parseCacheKey(cacheKey) {
480
453
  if (!cacheKey)
@@ -745,33 +718,23 @@ export function getOperationId() {
745
718
  function isDebugEnabled() {
746
719
  return config.logging.level === 'debug';
747
720
  }
748
- function buildContextMetadata() {
721
+ function mergeMetadata(meta) {
749
722
  const ctx = requestContext.getStore();
723
+ const hasMeta = meta && Object.keys(meta).length > 0;
750
724
  if (!ctx)
751
- return undefined;
725
+ return hasMeta ? meta : undefined;
752
726
  const { requestId, operationId, sessionId } = ctx;
753
727
  const includeSession = sessionId && isDebugEnabled();
754
728
  if (!requestId && !operationId && !includeSession)
755
- return undefined;
756
- const meta = {};
729
+ return hasMeta ? meta : undefined;
730
+ const contextMeta = {};
757
731
  if (requestId)
758
- meta['requestId'] = requestId;
732
+ contextMeta['requestId'] = requestId;
759
733
  if (operationId)
760
- meta['operationId'] = operationId;
734
+ contextMeta['operationId'] = operationId;
761
735
  if (includeSession)
762
- meta['sessionId'] = sessionId;
763
- return meta;
764
- }
765
- function mergeMetadata(meta) {
766
- const contextMeta = buildContextMetadata();
767
- const hasMeta = meta && Object.keys(meta).length > 0;
768
- if (!contextMeta && !hasMeta)
769
- return undefined;
770
- if (!contextMeta)
771
- return meta;
772
- if (!hasMeta)
773
- return contextMeta;
774
- return { ...contextMeta, ...meta };
736
+ contextMeta['sessionId'] = sessionId;
737
+ return hasMeta ? { ...contextMeta, ...meta } : contextMeta;
775
738
  }
776
739
  function formatMetadata(meta) {
777
740
  const merged = mergeMetadata(meta);
@@ -806,36 +769,25 @@ const LEVEL_PRIORITY = {
806
769
  function shouldLog(level) {
807
770
  return LEVEL_PRIORITY[level] >= LEVEL_PRIORITY[config.logging.level];
808
771
  }
772
+ const LOG_LEVEL_ALIASES = {
773
+ debug: 'debug',
774
+ info: 'info',
775
+ notice: 'info',
776
+ warning: 'warn',
777
+ warn: 'warn',
778
+ error: 'error',
779
+ critical: 'error',
780
+ alert: 'error',
781
+ emergency: 'error',
782
+ };
809
783
  function normalizeLogLevel(level) {
810
- switch (level.toLowerCase()) {
811
- case 'debug':
812
- return 'debug';
813
- case 'info':
814
- case 'notice':
815
- return 'info';
816
- case 'warning':
817
- case 'warn':
818
- return 'warn';
819
- case 'error':
820
- case 'critical':
821
- case 'alert':
822
- case 'emergency':
823
- return 'error';
824
- default:
825
- return undefined;
826
- }
827
- }
828
- function resolveMcpLogLevel(sessionId) {
829
- if (sessionId) {
830
- return sessionMcpLogLevels.get(sessionId) ?? config.logging.level;
831
- }
832
- return stdioMcpLogLevel ?? config.logging.level;
784
+ return LOG_LEVEL_ALIASES[level.toLowerCase()];
833
785
  }
834
786
  function shouldForwardMcpLog(level, sessionId) {
835
- return LEVEL_PRIORITY[level] >= LEVEL_PRIORITY[resolveMcpLogLevel(sessionId)];
836
- }
837
- function mapToMcpLevel(level) {
838
- return level === 'warn' ? 'warning' : level;
787
+ const mcpLevel = sessionId
788
+ ? (sessionMcpLogLevels.get(sessionId) ?? config.logging.level)
789
+ : (stdioMcpLogLevel ?? config.logging.level);
790
+ return LEVEL_PRIORITY[level] >= LEVEL_PRIORITY[mcpLevel];
839
791
  }
840
792
  function resolveErrorText(err) {
841
793
  if (err instanceof Error) {
@@ -886,7 +838,7 @@ function writeLog(level, message, meta) {
886
838
  try {
887
839
  server.server
888
840
  .sendLoggingMessage({
889
- level: mapToMcpLevel(level),
841
+ level: level === 'warn' ? 'warning' : level,
890
842
  logger: 'fetch-url-mcp',
891
843
  // Preserve existing behavior: MCP payload includes only message + provided meta (not ALS context meta).
892
844
  data: meta ? { message, ...meta } : message,
@@ -933,7 +885,10 @@ export function logError(message, error) {
933
885
  writeLog('error', message, errorMeta);
934
886
  }
935
887
  export function getMcpLogLevel(sessionId) {
936
- return resolveMcpLogLevel(sessionId);
888
+ if (sessionId) {
889
+ return sessionMcpLogLevels.get(sessionId) ?? config.logging.level;
890
+ }
891
+ return stdioMcpLogLevel ?? config.logging.level;
937
892
  }
938
893
  export function setLogLevel(level, sessionId) {
939
894
  const normalized = normalizeLogLevel(level);
@@ -990,11 +945,6 @@ function logRejectedSettledResults(results, message) {
990
945
  }
991
946
  }
992
947
  }
993
- function isSessionExpired(session, now, sessionTtlMs) {
994
- if (sessionTtlMs <= 0)
995
- return false;
996
- return now - session.lastSeen > sessionTtlMs;
997
- }
998
948
  class SessionCleanupLoop {
999
949
  store;
1000
950
  sessionTtlMs;
@@ -1075,18 +1025,6 @@ class SessionCleanupLoop {
1075
1025
  export function startSessionCleanupLoop(store, sessionTtlMs, options) {
1076
1026
  return new SessionCleanupLoop(store, sessionTtlMs, options?.onEvictSession, options?.cleanupIntervalMs).start();
1077
1027
  }
1078
- function moveSessionToEnd(sessions, sessionId, session) {
1079
- sessions.delete(sessionId);
1080
- sessions.set(sessionId, session);
1081
- }
1082
- function removeSessionById(sessions, sessionId) {
1083
- const session = sessions.get(sessionId);
1084
- sessions.delete(sessionId);
1085
- return session;
1086
- }
1087
- function isBlankSessionId(sessionId) {
1088
- return sessionId.length === 0;
1089
- }
1090
1028
  class InMemorySessionStore {
1091
1029
  sessionTtlMs;
1092
1030
  sessions = new Map();
@@ -1095,28 +1033,32 @@ class InMemorySessionStore {
1095
1033
  this.sessionTtlMs = sessionTtlMs;
1096
1034
  }
1097
1035
  get(sessionId) {
1098
- if (isBlankSessionId(sessionId))
1036
+ if (sessionId.length === 0)
1099
1037
  return undefined;
1100
1038
  return this.sessions.get(sessionId);
1101
1039
  }
1102
1040
  touch(sessionId) {
1103
- if (isBlankSessionId(sessionId))
1041
+ if (sessionId.length === 0)
1104
1042
  return;
1105
1043
  const session = this.sessions.get(sessionId);
1106
1044
  if (!session)
1107
1045
  return;
1108
1046
  session.lastSeen = Date.now();
1109
- moveSessionToEnd(this.sessions, sessionId, session);
1047
+ this.sessions.delete(sessionId);
1048
+ this.sessions.set(sessionId, session);
1110
1049
  }
1111
1050
  set(sessionId, entry) {
1112
- if (isBlankSessionId(sessionId))
1051
+ if (sessionId.length === 0)
1113
1052
  return;
1114
- moveSessionToEnd(this.sessions, sessionId, entry);
1053
+ this.sessions.delete(sessionId);
1054
+ this.sessions.set(sessionId, entry);
1115
1055
  }
1116
1056
  remove(sessionId) {
1117
- if (isBlankSessionId(sessionId))
1057
+ if (sessionId.length === 0)
1118
1058
  return undefined;
1119
- return removeSessionById(this.sessions, sessionId);
1059
+ const session = this.sessions.get(sessionId);
1060
+ this.sessions.delete(sessionId);
1061
+ return session;
1120
1062
  }
1121
1063
  size() {
1122
1064
  return this.sessions.size;
@@ -1141,10 +1083,13 @@ class InMemorySessionStore {
1141
1083
  const now = Date.now();
1142
1084
  const evicted = [];
1143
1085
  for (const [id, session] of this.sessions.entries()) {
1144
- if (!isSessionExpired(session, now, this.sessionTtlMs))
1145
- continue;
1146
- this.sessions.delete(id);
1147
- evicted.push(session);
1086
+ if (this.sessionTtlMs > 0 && now - session.lastSeen > this.sessionTtlMs) {
1087
+ this.sessions.delete(id);
1088
+ evicted.push(session);
1089
+ }
1090
+ else {
1091
+ break;
1092
+ }
1148
1093
  }
1149
1094
  return evicted;
1150
1095
  }
@@ -1152,49 +1097,40 @@ class InMemorySessionStore {
1152
1097
  const oldest = this.sessions.keys().next();
1153
1098
  if (oldest.done)
1154
1099
  return undefined;
1155
- return removeSessionById(this.sessions, oldest.value);
1100
+ const session = this.sessions.get(oldest.value);
1101
+ this.sessions.delete(oldest.value);
1102
+ return session;
1156
1103
  }
1157
1104
  }
1158
1105
  export function createSessionStore(sessionTtlMs) {
1159
1106
  return new InMemorySessionStore(sessionTtlMs);
1160
1107
  }
1161
- class SessionSlotTracker {
1162
- store;
1163
- slotReleased = false;
1164
- initialized = false;
1165
- constructor(store) {
1166
- this.store = store;
1167
- }
1168
- releaseSlot() {
1169
- if (this.slotReleased)
1170
- return;
1171
- this.slotReleased = true;
1172
- this.store.decrementInFlight();
1173
- }
1174
- markInitialized() {
1175
- this.initialized = true;
1176
- }
1177
- isInitialized() {
1178
- return this.initialized;
1179
- }
1180
- }
1181
1108
  export function createSlotTracker(store) {
1182
- return new SessionSlotTracker(store);
1183
- }
1184
- function currentLoad(store) {
1185
- return store.size() + store.inFlight();
1109
+ let slotReleased = false;
1110
+ let initialized = false;
1111
+ return {
1112
+ releaseSlot() {
1113
+ if (slotReleased)
1114
+ return;
1115
+ slotReleased = true;
1116
+ store.decrementInFlight();
1117
+ },
1118
+ markInitialized() {
1119
+ initialized = true;
1120
+ },
1121
+ isInitialized() {
1122
+ return initialized;
1123
+ },
1124
+ };
1186
1125
  }
1187
1126
  export function reserveSessionSlot(store, maxSessions) {
1188
1127
  if (maxSessions <= 0)
1189
1128
  return false;
1190
- if (currentLoad(store) >= maxSessions)
1129
+ if (store.size() + store.inFlight() >= maxSessions)
1191
1130
  return false;
1192
1131
  store.incrementInFlight();
1193
1132
  return true;
1194
1133
  }
1195
- function isAtCapacity(store, maxSessions) {
1196
- return currentLoad(store) >= maxSessions;
1197
- }
1198
1134
  export function ensureSessionCapacity({ store, maxSessions, evictOldest, }) {
1199
1135
  if (maxSessions <= 0)
1200
1136
  return false;
@@ -1207,5 +1143,5 @@ export function ensureSessionCapacity({ store, maxSessions, evictOldest, }) {
1207
1143
  return false;
1208
1144
  if (!evictOldest(store))
1209
1145
  return false;
1210
- return !isAtCapacity(store, maxSessions);
1146
+ return store.size() + store.inFlight() < maxSessions;
1211
1147
  }