@j0hanz/fetch-url-mcp 1.6.0 → 1.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (108) hide show
  1. package/README.md +507 -403
  2. package/dist/cli.d.ts +1 -0
  3. package/dist/cli.d.ts.map +1 -0
  4. package/dist/http/auth.d.ts +12 -0
  5. package/dist/http/auth.d.ts.map +1 -0
  6. package/dist/http/auth.js +85 -6
  7. package/dist/http/health.d.ts +1 -0
  8. package/dist/http/health.d.ts.map +1 -0
  9. package/dist/http/helpers.d.ts +1 -0
  10. package/dist/http/helpers.d.ts.map +1 -0
  11. package/dist/http/native.d.ts +1 -0
  12. package/dist/http/native.d.ts.map +1 -0
  13. package/dist/http/native.js +80 -63
  14. package/dist/http/rate-limit.d.ts +1 -0
  15. package/dist/http/rate-limit.d.ts.map +1 -0
  16. package/dist/index.d.ts +1 -0
  17. package/dist/index.d.ts.map +1 -0
  18. package/dist/lib/content.d.ts +3 -0
  19. package/dist/lib/content.d.ts.map +1 -0
  20. package/dist/lib/content.js +16 -11
  21. package/dist/lib/core.d.ts +6 -8
  22. package/dist/lib/core.d.ts.map +1 -0
  23. package/dist/lib/core.js +111 -97
  24. package/dist/lib/fetch-pipeline.d.ts +1 -1
  25. package/dist/lib/fetch-pipeline.d.ts.map +1 -0
  26. package/dist/lib/fetch-pipeline.js +54 -44
  27. package/dist/lib/http.d.ts +1 -0
  28. package/dist/lib/http.d.ts.map +1 -0
  29. package/dist/lib/mcp-tools.d.ts +45 -7
  30. package/dist/lib/mcp-tools.d.ts.map +1 -0
  31. package/dist/lib/mcp-tools.js +37 -6
  32. package/dist/lib/net-utils.d.ts +1 -0
  33. package/dist/lib/net-utils.d.ts.map +1 -0
  34. package/dist/lib/progress.d.ts +1 -0
  35. package/dist/lib/progress.d.ts.map +1 -0
  36. package/dist/lib/task-handlers.d.ts +9 -1
  37. package/dist/lib/task-handlers.d.ts.map +1 -0
  38. package/dist/lib/task-handlers.js +30 -38
  39. package/dist/lib/types.d.ts +4 -0
  40. package/dist/lib/types.d.ts.map +1 -0
  41. package/dist/lib/types.js +12 -1
  42. package/dist/lib/url.d.ts +3 -0
  43. package/dist/lib/url.d.ts.map +1 -0
  44. package/dist/lib/url.js +78 -151
  45. package/dist/lib/utils.d.ts +2 -2
  46. package/dist/lib/utils.d.ts.map +1 -0
  47. package/dist/lib/utils.js +60 -94
  48. package/dist/lib/zod.d.ts +3 -0
  49. package/dist/lib/zod.d.ts.map +1 -0
  50. package/dist/lib/zod.js +33 -0
  51. package/dist/prompts/index.d.ts +2 -1
  52. package/dist/prompts/index.d.ts.map +1 -0
  53. package/dist/prompts/index.js +2 -13
  54. package/dist/resources/index.d.ts +2 -1
  55. package/dist/resources/index.d.ts.map +1 -0
  56. package/dist/resources/index.js +5 -19
  57. package/dist/resources/instructions.d.ts +1 -0
  58. package/dist/resources/instructions.d.ts.map +1 -0
  59. package/dist/resources/instructions.js +2 -0
  60. package/dist/schemas/cache.d.ts +18 -0
  61. package/dist/schemas/cache.d.ts.map +1 -0
  62. package/dist/schemas/cache.js +19 -0
  63. package/dist/schemas/inputs.d.ts +1 -0
  64. package/dist/schemas/inputs.d.ts.map +1 -0
  65. package/dist/schemas/outputs.d.ts +6 -5
  66. package/dist/schemas/outputs.d.ts.map +1 -0
  67. package/dist/schemas/outputs.js +5 -9
  68. package/dist/server.d.ts +1 -0
  69. package/dist/server.d.ts.map +1 -0
  70. package/dist/server.js +9 -7
  71. package/dist/tasks/execution.d.ts +1 -0
  72. package/dist/tasks/execution.d.ts.map +1 -0
  73. package/dist/tasks/execution.js +3 -21
  74. package/dist/tasks/manager.d.ts +2 -6
  75. package/dist/tasks/manager.d.ts.map +1 -0
  76. package/dist/tasks/manager.js +2 -4
  77. package/dist/tasks/owner.d.ts +1 -0
  78. package/dist/tasks/owner.d.ts.map +1 -0
  79. package/dist/tasks/tool-registry.d.ts +2 -0
  80. package/dist/tasks/tool-registry.d.ts.map +1 -0
  81. package/dist/tasks/tool-registry.js +3 -0
  82. package/dist/tools/fetch-url.d.ts +4 -6
  83. package/dist/tools/fetch-url.d.ts.map +1 -0
  84. package/dist/tools/fetch-url.js +61 -59
  85. package/dist/tools/index.d.ts +1 -0
  86. package/dist/tools/index.d.ts.map +1 -0
  87. package/dist/transform/html-translators.d.ts +1 -0
  88. package/dist/transform/html-translators.d.ts.map +1 -0
  89. package/dist/transform/html-translators.js +5 -2
  90. package/dist/transform/metadata.d.ts +1 -0
  91. package/dist/transform/metadata.d.ts.map +1 -0
  92. package/dist/transform/metadata.js +1 -0
  93. package/dist/transform/{workers/shared.d.ts → shared.d.ts} +2 -1
  94. package/dist/transform/shared.d.ts.map +1 -0
  95. package/dist/transform/{workers/shared.js → shared.js} +1 -1
  96. package/dist/transform/transform.d.ts +1 -0
  97. package/dist/transform/transform.d.ts.map +1 -0
  98. package/dist/transform/transform.js +21 -14
  99. package/dist/transform/types.d.ts +1 -4
  100. package/dist/transform/types.d.ts.map +1 -0
  101. package/dist/transform/worker-pool.d.ts +3 -18
  102. package/dist/transform/worker-pool.d.ts.map +1 -0
  103. package/dist/transform/worker-pool.js +51 -167
  104. package/package.json +9 -6
  105. package/dist/transform/workers/transform-child.d.ts +0 -1
  106. package/dist/transform/workers/transform-child.js +0 -15
  107. package/dist/transform/workers/transform-worker.d.ts +0 -1
  108. package/dist/transform/workers/transform-worker.js +0 -13
@@ -4,7 +4,7 @@ import { randomUUID } from 'node:crypto';
4
4
  import { setInterval } from 'node:timers';
5
5
  import { ErrorCode, McpError } from '@modelcontextprotocol/sdk/types.js';
6
6
  import { config } from '../lib/core.js';
7
- import { RESOURCE_NOT_FOUND_ERROR_CODE, toError } from '../lib/utils.js';
7
+ import { toError } from '../lib/utils.js';
8
8
  import { createUnrefTimeout } from '../lib/utils.js';
9
9
  const DEFAULT_TTL_MS = 60_000;
10
10
  const MIN_TTL_MS = 1_000;
@@ -69,8 +69,6 @@ class TaskManager {
69
69
  applyTaskUpdate(task, updates) {
70
70
  Object.assign(task, updates);
71
71
  task.lastUpdatedAt = new Date().toISOString();
72
- // Slide TTL window on every activity so long-running tasks don't expire mid-flight.
73
- task._createdAtMs = Date.now();
74
72
  }
75
73
  cancelActiveTask(task, statusMessage) {
76
74
  this.applyTaskUpdate(task, {
@@ -322,7 +320,7 @@ class TaskManager {
322
320
  cleanup();
323
321
  this.removeWaiter(taskId, waiter);
324
322
  this.removeTask(taskId);
325
- rejectInContext(new McpError(RESOURCE_NOT_FOUND_ERROR_CODE, 'Task expired', {
323
+ rejectInContext(new McpError(ErrorCode.InvalidParams, 'Task expired', {
326
324
  taskId,
327
325
  }));
328
326
  });
@@ -30,3 +30,4 @@ export declare function resolveToolCallContext(extra?: HandlerExtra): ToolCallCo
30
30
  export declare function isServerResult(value: unknown): value is ServerResult;
31
31
  export declare function tryReadToolStructuredError(value: unknown): string | undefined;
32
32
  export {};
33
+ //# sourceMappingURL=owner.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"owner.d.ts","sourceRoot":"","sources":["../../src/tasks/owner.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oCAAoC,CAAC;AAEvE,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAOhE,UAAU,YAAY;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE;QAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACjD,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC5B,gBAAgB,CAAC,EAAE,CAAC,YAAY,EAAE,oBAAoB,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1E;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC5B,gBAAgB,CAAC,EAAE,CAAC,YAAY,EAAE,oBAAoB,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1E;AAED;;sDAEsD;AACtD,KAAK,SAAS,CAAC,CAAC,SAAS,MAAM,IAAI;KAChC,CAAC,IAAI,MAAM,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,SAAS,KAAK,GACnD,KAAK,GACL,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC;CAClC,CAAC;AAEF,wBAAgB,OAAO,CAAC,CAAC,SAAS,MAAM,EAAE,GAAG,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CAQ9D;AA2BD,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,OAAO,GAAG,YAAY,GAAG,SAAS,CAyB1E;AAED,wBAAgB,mBAAmB,CAAC,KAAK,CAAC,EAAE,YAAY,GAAG,MAAM,CAMhE;AAED,wBAAgB,sBAAsB,CAAC,KAAK,CAAC,EAAE,YAAY,GAAG,eAAe,CAQ5E;AAED,wBAAgB,cAAc,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,YAAY,CAIpE;AAED,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CAkB7E"}
@@ -9,3 +9,5 @@ export declare function registerTaskCapableTool<TArgs>(descriptor: TaskCapableTo
9
9
  export declare function unregisterTaskCapableTool(name: string): void;
10
10
  export declare function getTaskCapableTool(name: string): TaskCapableToolDescriptor | undefined;
11
11
  export declare function hasTaskCapableTool(name: string): boolean;
12
+ export declare function hasRegisteredTaskCapableTools(): boolean;
13
+ //# sourceMappingURL=tool-registry.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"tool-registry.d.ts","sourceRoot":"","sources":["../../src/tasks/tool-registry.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oCAAoC,CAAC;AAEvE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAE5D,MAAM,WAAW,yBAAyB,CAAC,KAAK,GAAG,OAAO;IACxD,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,EAAE,CAAC,IAAI,EAAE,OAAO,KAAK,KAAK,CAAC;IACzC,OAAO,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,EAAE,gBAAgB,KAAK,OAAO,CAAC,YAAY,CAAC,CAAC;CAC3E;AAID,wBAAgB,uBAAuB,CAAC,KAAK,EAC3C,UAAU,EAAE,yBAAyB,CAAC,KAAK,CAAC,GAC3C,IAAI,CAKN;AAED,wBAAgB,yBAAyB,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAE5D;AAED,wBAAgB,kBAAkB,CAChC,IAAI,EAAE,MAAM,GACX,yBAAyB,GAAG,SAAS,CAEvC;AAED,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAExD;AAED,wBAAgB,6BAA6B,IAAI,OAAO,CAEvD"}
@@ -11,3 +11,6 @@ export function getTaskCapableTool(name) {
11
11
  export function hasTaskCapableTool(name) {
12
12
  return taskCapableTools.has(name);
13
13
  }
14
+ export function hasRegisteredTaskCapableTools() {
15
+ return taskCapableTools.size > 0;
16
+ }
@@ -1,12 +1,9 @@
1
1
  import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
2
2
  import type { ContentBlock } from '@modelcontextprotocol/sdk/types.js';
3
+ import { z } from 'zod';
3
4
  import { type ToolHandlerExtra } from '../lib/mcp-tools.js';
4
- interface FetchUrlInput {
5
- url: string;
6
- skipNoiseRemoval?: boolean | undefined;
7
- forceRefresh?: boolean | undefined;
8
- maxInlineChars?: number | undefined;
9
- }
5
+ import { fetchUrlInputSchema } from '../schemas/inputs.js';
6
+ type FetchUrlInput = z.infer<typeof fetchUrlInputSchema>;
10
7
  type ToolContentBlockUnion = ContentBlock;
11
8
  interface ToolResponseBase {
12
9
  [key: string]: unknown;
@@ -26,3 +23,4 @@ export declare function fetchUrlToolHandler(input: FetchUrlInput, extra?: ToolHa
26
23
  export declare function withRequestContextIfMissing<TParams, TResult, TExtra = unknown>(handler: (params: TParams, extra?: TExtra) => Promise<TResult>): (params: TParams, extra?: TExtra) => Promise<TResult>;
27
24
  export declare function registerTools(server: McpServer): void;
28
25
  export {};
26
+ //# sourceMappingURL=fetch-url.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"fetch-url.d.ts","sourceRoot":"","sources":["../../src/tools/fetch-url.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACV,SAAS,EAEV,MAAM,yCAAyC,CAAC;AACjD,OAAO,KAAK,EACV,YAAY,EAEb,MAAM,oCAAoC,CAAC;AAE5C,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAwBxB,OAAO,EAGL,KAAK,gBAAgB,EACtB,MAAM,qBAAqB,CAAC;AAG7B,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAS3D,KAAK,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAEzD,KAAK,qBAAqB,GAAG,YAAY,CAAC;AAE1C,UAAU,gBAAgB;IACxB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;IACvB,OAAO,EAAE,qBAAqB,EAAE,CAAC;IACjC,iBAAiB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,CAAC;IACxD,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,eAAO,MAAM,mBAAmB,cAAc,CAAC;AA8T/C,wBAAsB,mBAAmB,CACvC,KAAK,EAAE,aAAa,EACpB,KAAK,CAAC,EAAE,gBAAgB,GACvB,OAAO,CAAC,gBAAgB,CAAC,CAK3B;AAgDD;;;;;;GAMG;AACH,wBAAgB,2BAA2B,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,EAC5E,OAAO,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,GAC7D,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAmBvD;AAwBD,wBAAgB,aAAa,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI,CAwCrD"}
@@ -7,6 +7,7 @@ import { handleToolError } from '../lib/mcp-tools.js';
7
7
  import { appendTruncationMarker, markdownTransform, parseCachedMarkdownResult, performSharedFetch, readNestedRecord, serializeMarkdownResult, TRUNCATION_MARKER, withSignal, } from '../lib/mcp-tools.js';
8
8
  import { createProgressReporter, } from '../lib/mcp-tools.js';
9
9
  import { isAbortError, isObject, toError } from '../lib/utils.js';
10
+ import { formatZodError } from '../lib/zod.js';
10
11
  import { fetchUrlInputSchema } from '../schemas/inputs.js';
11
12
  import { fetchUrlOutputSchema } from '../schemas/outputs.js';
12
13
  import { registerTaskCapableTool, unregisterTaskCapableTool, } from '../tasks/tool-registry.js';
@@ -26,7 +27,6 @@ const TOOL_ICON = {
26
27
  src: 'data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCAyNCAyNCIgZmlsbD0ibm9uZSIgc3Ryb2tlPSJjdXJyZW50Q29sb3IiIHN0cm9rZS13aWR0aD0iMiIgc3Ryb2tlLWxpbmVjYXA9InJvdW5kIiBzdHJva2UtbGluZWpvaW49InJvdW5kIj48cGF0aCBkPSJNMjEgMTV2NGEyIDIgMCAwIDEtMiAySDVhMiAyIDAgMCAxLTItMnYtNCIvPjxwb2x5bGluZSBwb2ludHM9IjcgMTAgMTIgMTUgMTcgMTAiLz48bGluZSB4MT0iMTIiIHkxPSIxNSIgeDI9IjEyIiB5Mj0iMyIvPjwvc3ZnPg==',
27
28
  mimeType: 'image/svg+xml',
28
29
  };
29
- const JSON_SCHEMA_DRAFT_2020_12_URI = 'https://json-schema.org/draft/2020-12/schema';
30
30
  /* -------------------------------------------------------------------------------------------------
31
31
  * Tool response builders
32
32
  * ------------------------------------------------------------------------------------------------- */
@@ -121,6 +121,42 @@ function buildResponse(pipeline, inlineResult, inputUrl) {
121
121
  /* -------------------------------------------------------------------------------------------------
122
122
  * fetch-url tool implementation
123
123
  * ------------------------------------------------------------------------------------------------- */
124
+ function isCodeHost(host) {
125
+ return (host === 'github.com' || host === 'gitlab.com' || host === 'bitbucket.org');
126
+ }
127
+ function summarizeCodeHostPath(host, parts) {
128
+ if (!isCodeHost(host) || parts.length < 2)
129
+ return null;
130
+ const p0 = parts[0] ?? '';
131
+ const p1 = parts[1] ?? '';
132
+ return `${host}/${p0}/${p1}`;
133
+ }
134
+ function summarizeWikipediaPath(parts) {
135
+ if (parts[0] !== 'wiki' || parts.length < 2)
136
+ return null;
137
+ const p1 = parts[1] ?? '';
138
+ return `wikipedia.org/${p1}`;
139
+ }
140
+ function truncatePathSegment(segment, max = 20) {
141
+ if (segment.length <= max)
142
+ return segment;
143
+ return `${segment.substring(0, max - 3)}...`;
144
+ }
145
+ function summarizeUrlPath(host, parts) {
146
+ const codeHostSummary = summarizeCodeHostPath(host, parts);
147
+ if (codeHostSummary)
148
+ return codeHostSummary;
149
+ if (host.endsWith('wikipedia.org')) {
150
+ const wikipediaSummary = summarizeWikipediaPath(parts);
151
+ if (wikipediaSummary)
152
+ return wikipediaSummary;
153
+ }
154
+ const basename = truncatePathSegment(parts.at(-1) ?? '');
155
+ if (parts.length === 1) {
156
+ return `${host}/${basename}`;
157
+ }
158
+ return basename ? `${host}/…/${basename}` : host;
159
+ }
124
160
  function getUrlContext(urlStr) {
125
161
  try {
126
162
  const u = new URL(urlStr);
@@ -131,82 +167,61 @@ function getUrlContext(urlStr) {
131
167
  const parts = path.split('/').filter(Boolean);
132
168
  if (parts.length === 0)
133
169
  return host;
134
- // Special case for GitHub/GitLab/Bitbucket
135
- if (host === 'github.com' ||
136
- host === 'gitlab.com' ||
137
- host === 'bitbucket.org') {
138
- if (parts.length >= 2) {
139
- const p0 = parts[0] ?? '';
140
- const p1 = parts[1] ?? '';
141
- return `${host}/${p0}/${p1}`;
142
- }
143
- }
144
- // Special case for Wikipedia
145
- if (host.endsWith('wikipedia.org') &&
146
- parts[0] === 'wiki' &&
147
- parts.length >= 2) {
148
- const p1 = parts[1] ?? '';
149
- return `wikipedia.org/${p1}`;
150
- }
151
- let basename = parts.pop() ?? '';
152
- if (basename && basename.length > 20) {
153
- basename = `${basename.substring(0, 17)}...`;
154
- }
155
- if (parts.length === 0) {
156
- return `${host}/${basename}`;
157
- }
158
- return basename ? `${host}/…/${basename}` : host;
170
+ return summarizeUrlPath(host, parts);
159
171
  }
160
172
  catch {
161
173
  return 'unknown';
162
174
  }
163
175
  }
164
- async function fetchPipeline(url, signal, progress, skipNoiseRemoval, forceRefresh, maxInlineChars) {
165
- const contextStr = getUrlContext(url);
166
- return performSharedFetch({
176
+ function buildFetchOptions(url, signal, progress, skipNoiseRemoval, forceRefresh, maxInlineChars) {
177
+ return {
167
178
  url,
168
179
  ...withSignal(signal),
169
180
  ...(skipNoiseRemoval ? { cacheVary: { skipNoiseRemoval: true } } : {}),
170
181
  ...(forceRefresh ? { forceRefresh: true } : {}),
171
182
  ...(maxInlineChars !== undefined ? { maxInlineChars } : {}),
172
183
  transform: async ({ buffer, encoding, truncated }, normalizedUrl) => {
173
- reportFetchProgress(progress, 2, contextStr, 'converting to Markdown');
184
+ reportProgress(progress, 2, 'Parsing HTML Markdown');
174
185
  return markdownTransform({ buffer, encoding, ...(truncated ? { truncated } : {}) }, normalizedUrl, signal, skipNoiseRemoval);
175
186
  },
176
187
  serialize: serializeMarkdownResult,
177
188
  deserialize: parseCachedMarkdownResult,
178
- });
189
+ };
179
190
  }
180
- function buildFetchProgressMessage(context, state) {
181
- if (state === 'completed' || state === 'cancelled' || state === 'failed') {
182
- return `fetch-url: ${context} • ${state}`;
183
- }
184
- return `fetch-url: ${context} [${state}]`;
191
+ async function fetchPipeline(url, signal, progress, skipNoiseRemoval, forceRefresh, maxInlineChars) {
192
+ return performSharedFetch(buildFetchOptions(url, signal, progress, skipNoiseRemoval, forceRefresh, maxInlineChars));
193
+ }
194
+ function formatContentSize(chars) {
195
+ if (chars < 1000)
196
+ return `${chars} chars`;
197
+ if (chars < 1_000_000)
198
+ return `${(chars / 1024).toFixed(1)} KB`;
199
+ return `${(chars / (1024 * 1024)).toFixed(1)} MB`;
185
200
  }
186
- function reportFetchProgress(progress, step, context, state) {
201
+ function reportProgress(progress, step, message) {
187
202
  if (!progress)
188
203
  return;
189
- progress.report(step, buildFetchProgressMessage(context, state));
204
+ progress.report(step, message);
190
205
  }
191
206
  async function executeFetch(input, extra) {
192
207
  const { url } = input;
193
208
  const signal = buildToolAbortSignal(extra?.signal);
194
209
  const progress = createProgressReporter(extra);
195
- const contextStr = getUrlContext(url);
196
- reportFetchProgress(progress, 0, contextStr, 'starting');
210
+ const context = getUrlContext(url);
197
211
  logDebug('Fetching URL', { url });
198
212
  try {
199
- reportFetchProgress(progress, 1, contextStr, 'fetching HTML');
213
+ reportProgress(progress, 1, `Fetching ${context}`);
200
214
  const { pipeline, inlineResult } = await fetchPipeline(url, signal, progress, input.skipNoiseRemoval, input.forceRefresh, input.maxInlineChars);
201
215
  if (pipeline.fromCache) {
202
- reportFetchProgress(progress, 3, contextStr, 'loaded from cache');
216
+ reportProgress(progress, 3, 'Loaded from cache');
203
217
  }
204
- reportFetchProgress(progress, 4, contextStr, 'completed');
218
+ const size = formatContentSize(inlineResult.contentSize);
219
+ reportProgress(progress, 4, `Done — ${size}`);
205
220
  return buildResponse(pipeline, inlineResult, url);
206
221
  }
207
222
  catch (error) {
208
223
  const isAbort = isAbortError(error);
209
- reportFetchProgress(progress, 4, contextStr, isAbort ? 'cancelled' : 'failed');
224
+ reportProgress(progress, 4, isAbort ? 'Cancelled' : 'Failed');
210
225
  throw error;
211
226
  }
212
227
  }
@@ -216,21 +231,12 @@ export async function fetchUrlToolHandler(input, extra) {
216
231
  return handleToolError(error, input.url, 'Failed to fetch URL');
217
232
  });
218
233
  }
219
- function withJsonSchema202012(schema) {
220
- if (typeof schema['$schema'] === 'string')
221
- return schema;
222
- return {
223
- $schema: JSON_SCHEMA_DRAFT_2020_12_URI,
224
- ...schema,
225
- };
226
- }
227
234
  const TOOL_DEFINITION = {
228
235
  name: FETCH_URL_TOOL_NAME,
229
236
  title: 'Fetch URL',
230
237
  description: FETCH_URL_TOOL_DESCRIPTION,
231
238
  inputSchema: fetchUrlInputSchema,
232
- // Explicitly mark JSON Schema dialect for MCP clients and static reviews.
233
- outputSchema: withJsonSchema202012(z.toJSONSchema(fetchUrlOutputSchema)),
239
+ outputSchema: z.toJSONSchema(fetchUrlOutputSchema),
234
240
  handler: fetchUrlToolHandler,
235
241
  execution: {
236
242
  taskSupport: 'optional',
@@ -299,11 +305,7 @@ export function registerTools(server) {
299
305
  parseArguments: (args) => {
300
306
  const parsed = fetchUrlInputSchema.safeParse(args);
301
307
  if (!parsed.success) {
302
- const flat = z.flattenError(parsed.error);
303
- const details = Object.entries(flat.fieldErrors)
304
- .map(([k, v]) => `${k}: ${v.join(', ')}`)
305
- .join('; ') || flat.formErrors.join('; ');
306
- throw new McpError(ErrorCode.InvalidParams, `Invalid arguments for ${FETCH_URL_TOOL_NAME}: ${details}`);
308
+ throw new McpError(ErrorCode.InvalidParams, `Invalid arguments for ${FETCH_URL_TOOL_NAME}: ${formatZodError(parsed.error)}`);
307
309
  }
308
310
  return parsed.data;
309
311
  },
@@ -1,2 +1,3 @@
1
1
  import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
2
2
  export declare function registerAllTools(server: McpServer): void;
3
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/tools/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AAIzE,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI,CAExD"}
@@ -1 +1,2 @@
1
1
  export declare function translateHtmlFragmentToMarkdown(html: string): string;
2
+ //# sourceMappingURL=html-translators.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"html-translators.d.ts","sourceRoot":"","sources":["../../src/transform/html-translators.ts"],"names":[],"mappings":"AAghBA,wBAAgB,+BAA+B,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAEpE"}
@@ -157,9 +157,9 @@ function resolveImageSrc(getAttribute) {
157
157
  if (url)
158
158
  return url;
159
159
  }
160
- // If the only available src is a data URI, we choose to omit it rather than include the raw data in the alt text or URL, as data URIs can be very long and are not useful in Markdown output.
160
+ // If the only available src is a data URI, omit the image URL entirely.
161
161
  if (isDataUri(srcRaw))
162
- return '[data URI removed]';
162
+ return '';
163
163
  return '';
164
164
  }
165
165
  function deriveAltFromImageUrl(src) {
@@ -203,6 +203,9 @@ function buildImageTranslator(ctx) {
203
203
  : undefined;
204
204
  const src = resolveImageSrc(getAttribute);
205
205
  const existingAlt = getAttribute?.('alt') ?? '';
206
+ if (!src) {
207
+ return { content: existingAlt.trim() };
208
+ }
206
209
  const alt = existingAlt.trim() || deriveAltFromImageUrl(src);
207
210
  const markdown = `![${alt}](${src})`;
208
211
  return { content: markdown };
@@ -2,3 +2,4 @@ import type { ExtractedMetadata } from './types.js';
2
2
  export declare function extractMetadata(document: Document, baseUrl?: string): ExtractedMetadata;
3
3
  export declare function extractMetadataFromHead(html: string, baseUrl?: string): ExtractedMetadata | null;
4
4
  export declare function mergeMetadata(early: ExtractedMetadata | null, late: ExtractedMetadata): ExtractedMetadata;
5
+ //# sourceMappingURL=metadata.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"metadata.d.ts","sourceRoot":"","sources":["../../src/transform/metadata.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAyKpD,wBAAgB,eAAe,CAC7B,QAAQ,EAAE,QAAQ,EAClB,OAAO,CAAC,EAAE,MAAM,GACf,iBAAiB,CAenB;AAED,wBAAgB,uBAAuB,CACrC,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE,MAAM,GACf,iBAAiB,GAAG,IAAI,CAY1B;AAED,wBAAgB,aAAa,CAC3B,KAAK,EAAE,iBAAiB,GAAG,IAAI,EAC/B,IAAI,EAAE,iBAAiB,GACtB,iBAAiB,CAmBnB"}
@@ -171,6 +171,7 @@ export function mergeMetadata(early, late) {
171
171
  'description',
172
172
  'author',
173
173
  'image',
174
+ 'favicon',
174
175
  'publishedAt',
175
176
  'modifiedAt',
176
177
  ];
@@ -1,7 +1,8 @@
1
- import type { MarkdownTransformResult, TransformOptions, TransformWorkerOutgoingMessage } from '../types.js';
1
+ import type { MarkdownTransformResult, TransformOptions, TransformWorkerOutgoingMessage } from './types.js';
2
2
  interface WorkerMessageHandlerOptions {
3
3
  sendMessage: (message: TransformWorkerOutgoingMessage) => void;
4
4
  runTransform: (html: string, url: string, options: TransformOptions) => MarkdownTransformResult;
5
5
  }
6
6
  export declare function createTransformMessageHandler(options: WorkerMessageHandlerOptions): (raw: unknown) => void;
7
7
  export {};
8
+ //# sourceMappingURL=shared.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"shared.d.ts","sourceRoot":"","sources":["../../src/transform/shared.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACV,uBAAuB,EACvB,gBAAgB,EAChB,8BAA8B,EAE/B,MAAM,YAAY,CAAC;AAEpB,UAAU,2BAA2B;IACnC,WAAW,EAAE,CAAC,OAAO,EAAE,8BAA8B,KAAK,IAAI,CAAC;IAC/D,YAAY,EAAE,CACZ,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,KACtB,uBAAuB,CAAC;CAC9B;AAkGD,wBAAgB,6BAA6B,CAC3C,OAAO,EAAE,2BAA2B,GACnC,CAAC,GAAG,EAAE,OAAO,KAAK,IAAI,CAgFxB"}
@@ -1,4 +1,4 @@
1
- import { FetchError, getErrorMessage } from '../../lib/utils.js';
1
+ import { FetchError, getErrorMessage } from '../lib/utils.js';
2
2
  function isTransformMessage(message) {
3
3
  if (!message || typeof message !== 'object')
4
4
  return false;
@@ -34,3 +34,4 @@ type TransformExecutionOptions = TransformOptions & {
34
34
  export declare function transformHtmlToMarkdown(html: string, url: string, options: TransformOptions): Promise<MarkdownTransformResult>;
35
35
  export declare function transformBufferToMarkdown(htmlBuffer: Uint8Array, url: string, options: TransformExecutionOptions): Promise<MarkdownTransformResult>;
36
36
  export {};
37
+ //# sourceMappingURL=transform.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"transform.d.ts","sourceRoot":"","sources":["../../src/transform/transform.ts"],"names":[],"mappings":"AAsCA,OAAO,KAAK,EACV,gBAAgB,EAChB,iBAAiB,EACjB,gBAAgB,EAChB,uBAAuB,EACvB,aAAa,EACb,gBAAgB,EAChB,qBAAqB,EAEtB,MAAM,YAAY,CAAC;AAqCpB,UAAU,WAAW;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;CACnB;AA4ID,wBAAgB,mBAAmB,CACjC,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,EACb,MAAM,CAAC,EAAE,WAAW,GACnB,qBAAqB,GAAG,IAAI,CAE9B;AAED,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,qBAAqB,GAAG,IAAI,EACrC,OAAO,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,OAAO,CAAA;CAAE,GAChC,MAAM,CAER;AAkUD,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,GAAE;IAAE,cAAc,CAAC,EAAE,OAAO,CAAC;IAAC,MAAM,CAAC,EAAE,WAAW,CAAA;CAExD,GACA,gBAAgB,CAGlB;AAqPD,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,QAAQ,CAAC,EAAE,aAAa,EACxB,OAAO,CAAC,EAAE;IACR,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B,GACA,MAAM,CAsBR;AAuKD,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,gBAAgB,GAAG,IAAI,EAChC,sBAAsB,EAAE,MAAM,GAAG,QAAQ,GACxC,OAAO,CAQT;AA6DD,wBAAgB,gCAAgC,CAC9C,OAAO,EAAE,gBAAgB,GAAG,IAAI,GAC/B,OAAO,IAAI,gBAAgB,CAE7B;AAED,wBAAgB,0BAA0B,CACxC,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GAAG,IAAI,EAChC,aAAa,EAAE,iBAAiB,EAChC,wBAAwB,EAAE,OAAO,EACjC,eAAe,EAAE,OAAO,GACvB,aAAa,GAAG,SAAS,CAqB3B;AAmQD,wBAAgB,gCAAgC,CAC9C,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GACxB,uBAAuB,CA8CzB;AAED,UAAU,kBAAkB;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,wBAAgB,qBAAqB,IAAI,kBAAkB,GAAG,IAAI,CAEjE;AAED,wBAAsB,2BAA2B,IAAI,OAAO,CAAC,IAAI,CAAC,CAEjE;AAED,KAAK,yBAAyB,GAAG,gBAAgB,GAAG;IAAE,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAoI1E,wBAAsB,uBAAuB,CAC3C,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,uBAAuB,CAAC,CAElC;AAED,wBAAsB,yBAAyB,CAC7C,UAAU,EAAE,UAAU,EACtB,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,yBAAyB,GACjC,OAAO,CAAC,uBAAuB,CAAC,CAElC"}
@@ -3,8 +3,7 @@ import diagnosticsChannel from 'node:diagnostics_channel';
3
3
  import { performance } from 'node:perf_hooks';
4
4
  import { isProbablyReaderable, Readability } from '@mozilla/readability';
5
5
  import { parseHTML } from 'linkedom';
6
- import { removeNoiseFromHtml } from '../lib/content.js';
7
- import { addSourceToMarkdown, buildMetadataFooter, cleanupMarkdownArtifacts, extractTitleFromRawMarkdown, isRawTextContent, } from '../lib/content.js';
6
+ import { addSourceToMarkdown, buildMetadataFooter, cleanupMarkdownArtifacts, extractTitleFromRawMarkdown, isRawTextContent, prepareDocumentForMarkdown, removeNoiseFromHtml, serializeDocumentForMarkdown, } from '../lib/content.js';
8
7
  import { config } from '../lib/core.js';
9
8
  import { getOperationId, getRequestId, logDebug, logError, logInfo, logWarn, redactUrl, } from '../lib/core.js';
10
9
  import { isRawTextContentUrl } from '../lib/http.js';
@@ -14,7 +13,6 @@ import { isObject } from '../lib/utils.js';
14
13
  import { translateHtmlFragmentToMarkdown } from './html-translators.js';
15
14
  import { extractMetadata, extractMetadataFromHead, mergeMetadata, } from './metadata.js';
16
15
  import { getOrCreateWorkerPool, getWorkerPoolStats, shutdownWorkerPool, } from './worker-pool.js';
17
- const utf8Decoder = new TextDecoder('utf-8');
18
16
  function decodeInput(input, encoding) {
19
17
  if (typeof input === 'string')
20
18
  return input;
@@ -22,13 +20,15 @@ function decodeInput(input, encoding) {
22
20
  if (!normalizedEncoding ||
23
21
  normalizedEncoding === 'utf-8' ||
24
22
  normalizedEncoding === 'utf8') {
25
- return utf8Decoder.decode(input);
23
+ const buf = Buffer.isBuffer(input) ? input : Buffer.from(input);
24
+ return buf.toString('utf8');
26
25
  }
27
26
  try {
28
27
  return new TextDecoder(normalizedEncoding).decode(input);
29
28
  }
30
29
  catch {
31
- return utf8Decoder.decode(input);
30
+ const buf = Buffer.isBuffer(input) ? input : Buffer.from(input);
31
+ return buf.toString('utf8');
32
32
  }
33
33
  }
34
34
  function asError(value) {
@@ -888,13 +888,20 @@ function buildContentSource(params) {
888
888
  };
889
889
  }
890
890
  if (document) {
891
- const cleanedHtml = skipNoiseRemoval
892
- ? html
893
- : removeNoiseFromHtml(html, document, url, signal);
891
+ if (skipNoiseRemoval) {
892
+ return {
893
+ ...base,
894
+ sourceHtml: html,
895
+ title: extractedMeta.title,
896
+ skipNoiseRemoval: true,
897
+ document,
898
+ };
899
+ }
900
+ prepareDocumentForMarkdown(document, url, signal);
894
901
  const contentRoot = findContentRoot(document);
895
902
  return {
896
903
  ...base,
897
- sourceHtml: contentRoot ?? cleanedHtml,
904
+ sourceHtml: contentRoot ?? serializeDocumentForMarkdown(document, html),
898
905
  title: extractedMeta.title,
899
906
  skipNoiseRemoval: true,
900
907
  document,
@@ -912,7 +919,7 @@ function resolveContentSource(params) {
912
919
  ...(params.signal ? { signal: params.signal } : {}),
913
920
  ...(params.inputTruncated ? { inputTruncated: true } : {}),
914
921
  });
915
- const useArticleContent = article
922
+ const useArticleContent = !params.skipNoiseRemoval && article
916
923
  ? shouldUseArticleContent(article, document)
917
924
  : false;
918
925
  return buildContentSource({
@@ -965,11 +972,11 @@ function hasBinaryIndicators(content) {
965
972
  if (content.includes('\x00'))
966
973
  return true;
967
974
  const sampleSize = Math.min(content.length, 2000);
968
- const sample = content.slice(0, sampleSize);
969
975
  let replacementCount = 0;
970
- for (const char of sample) {
971
- if (char === REPLACEMENT_CHAR)
972
- replacementCount++;
976
+ let i = -1;
977
+ while ((i = content.indexOf(REPLACEMENT_CHAR, i + 1)) !== -1 &&
978
+ i < sampleSize) {
979
+ replacementCount++;
973
980
  }
974
981
  return replacementCount > sampleSize * BINARY_INDICATOR_THRESHOLD;
975
982
  }
@@ -101,10 +101,6 @@ export interface TransformWorkerTransformMessage {
101
101
  skipNoiseRemoval?: boolean | undefined;
102
102
  inputTruncated?: boolean | undefined;
103
103
  }
104
- export interface TransformWorkerCancelMessage {
105
- type: 'cancel';
106
- id: string;
107
- }
108
104
  export interface TransformWorkerCancelledMessage {
109
105
  type: 'cancelled';
110
106
  id: string;
@@ -127,3 +123,4 @@ export interface TransformWorkerErrorMessage {
127
123
  }
128
124
  export type TransformWorkerOutgoingMessage = TransformWorkerResultMessage | TransformWorkerErrorMessage | TransformWorkerCancelledMessage;
129
125
  export {};
126
+ //# sourceMappingURL=types.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/transform/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,UAAU,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,gBAAgB,GAAG,IAAI,CAAC;IACjC,QAAQ,EAAE,iBAAiB,CAAC;CAC7B;AAED,UAAU,eAAe;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,SAAS,EAAE,OAAO,CAAC;IACnB,QAAQ,CAAC,EAAE,iBAAiB,CAAC;CAC9B;AAED;;GAEG;AACH,MAAM,WAAW,uBAAwB,SAAQ,eAAe;IAC9D,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC;CAC3B;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,eAAe,EAAE,OAAO,CAAC;IACzB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,CAAC,EAAE,CAAC,CAAC;IACL,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;CACjC;AAED;;GAEG;AACH,MAAM,WAAW,+BAA+B;IAC9C,IAAI,EAAE,WAAW,CAAC;IAClB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B,UAAU,CAAC,EAAE,UAAU,GAAG,SAAS,CAAC;IACpC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,eAAe,EAAE,OAAO,CAAC;IACzB,gBAAgB,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IACvC,cAAc,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CACtC;AAED,MAAM,WAAW,+BAA+B;IAC9C,IAAI,EAAE,WAAW,CAAC;IAClB,EAAE,EAAE,MAAM,CAAC;CACZ;AAED,MAAM,WAAW,4BAA4B;IAC3C,IAAI,EAAE,QAAQ,CAAC;IACf,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,eAAe,CAAC;CACzB;AAED,MAAM,WAAW,2BAA2B;IAC1C,IAAI,EAAE,OAAO,CAAC;IACd,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE;QACL,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,MAAM,CAAC;QAChB,GAAG,EAAE,MAAM,CAAC;QACZ,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACnC,CAAC;CACH;AAED,MAAM,MAAM,8BAA8B,GACtC,4BAA4B,GAC5B,2BAA2B,GAC3B,+BAA+B,CAAC"}
@@ -1,19 +1,4 @@
1
- import { type Transferable as NodeTransferable } from 'node:worker_threads';
2
- import type { MarkdownTransformResult, TransformWorkerCancelMessage, TransformWorkerTransformMessage } from './types.js';
3
- type TransformWorkerMessage = TransformWorkerTransformMessage | TransformWorkerCancelMessage;
4
- interface WorkerHost {
5
- kind: 'thread' | 'process';
6
- supportsTransferList: boolean;
7
- threadId?: number;
8
- pid?: number;
9
- postMessage: (message: TransformWorkerMessage, transferList?: NodeTransferable[]) => void;
10
- terminate: () => Promise<void>;
11
- unref: () => void;
12
- onMessage: (handler: (raw: unknown) => void) => void;
13
- onError: (handler: (error: unknown) => void) => void;
14
- onExit: (handler: (code: number | null, signal: NodeJS.Signals | null) => void) => void;
15
- }
16
- type WorkerSpawner = (workerIndex: number, name: string) => WorkerHost;
1
+ import type { MarkdownTransformResult } from './types.js';
17
2
  interface TransformWorkerPool {
18
3
  transform(html: string, url: string, options: {
19
4
  includeMetadata: boolean;
@@ -38,10 +23,9 @@ declare class WorkerPool implements TransformWorkerPool {
38
23
  private readonly cancelAcks;
39
24
  private readonly timeoutMs;
40
25
  private readonly queueMax;
41
- private readonly spawnWorkerImpl;
42
26
  private closed;
43
27
  private taskIdSeq;
44
- constructor(size: number, timeoutMs: number, spawnWorker: WorkerSpawner);
28
+ constructor(size: number, timeoutMs: number);
45
29
  transform(html: string, url: string, options: {
46
30
  includeMetadata: boolean;
47
31
  signal?: AbortSignal;
@@ -90,3 +74,4 @@ export declare function getWorkerPoolStats(): {
90
74
  } | null;
91
75
  export declare function shutdownWorkerPool(): Promise<void>;
92
76
  export {};
77
+ //# sourceMappingURL=worker-pool.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"worker-pool.d.ts","sourceRoot":"","sources":["../../src/transform/worker-pool.ts"],"names":[],"mappings":"AAqBA,OAAO,KAAK,EACV,uBAAuB,EAKxB,MAAM,YAAY,CAAC;AAwLpB,UAAU,mBAAmB;IAC3B,SAAS,CACP,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE;QACP,eAAe,EAAE,OAAO,CAAC;QACzB,MAAM,CAAC,EAAE,WAAW,CAAC;QACrB,gBAAgB,CAAC,EAAE,OAAO,CAAC;QAC3B,cAAc,CAAC,EAAE,OAAO,CAAC;KAC1B,GACA,OAAO,CAAC,uBAAuB,CAAC,CAAC;IACpC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACvB,aAAa,IAAI,MAAM,CAAC;IACxB,gBAAgB,IAAI,MAAM,CAAC;IAC3B,WAAW,IAAI,MAAM,CAAC;CACvB;AAkBD,cAAM,UAAW,YAAW,mBAAmB;IAC7C,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAkC;IAExE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkC;IAC1D,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAoB;IAChD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAoB;IAEhD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAqB;IAC3C,OAAO,CAAC,SAAS,CAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAmC;IAC5D,OAAO,CAAC,QAAQ,CAAC,UAAU,CAOvB;IAEJ,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,SAAS,CAAK;gBAEV,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM;IASrC,SAAS,CACb,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE;QACP,eAAe,EAAE,OAAO,CAAC;QACzB,MAAM,CAAC,EAAE,WAAW,CAAC;QACrB,gBAAgB,CAAC,EAAE,OAAO,CAAC;QAC3B,cAAc,CAAC,EAAE,OAAO,CAAC;KAC1B,GACA,OAAO,CAAC,uBAAuB,CAAC;IAC7B,SAAS,CACb,UAAU,EAAE,UAAU,EACtB,GAAG,EAAE,MAAM,EACX,OAAO,EAAE;QACP,eAAe,EAAE,OAAO,CAAC;QACzB,MAAM,CAAC,EAAE,WAAW,CAAC;QACrB,gBAAgB,CAAC,EAAE,OAAO,CAAC;QAC3B,cAAc,CAAC,EAAE,OAAO,CAAC;QACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;KACnB,GACA,OAAO,CAAC,uBAAuB,CAAC;IAoCnC,aAAa,IAAI,MAAM;IAKvB,gBAAgB,IAAI,MAAM;IAI1B,WAAW,IAAI,MAAM;IAIrB,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAWpB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAiC5B,OAAO,CAAC,UAAU;IAIlB,OAAO,CAAC,iBAAiB;IAoDzB,OAAO,CAAC,aAAa;IAsCrB,OAAO,CAAC,gBAAgB;IAOxB,OAAO,CAAC,gBAAgB;YAyBV,aAAa;IA2B3B,OAAO,CAAC,kBAAkB;IAY1B,OAAO,CAAC,WAAW;IAmCnB,OAAO,CAAC,cAAc;IAuBtB,OAAO,CAAC,aAAa;IAYrB,OAAO,CAAC,eAAe;IAuDvB,OAAO,CAAC,YAAY;IAWpB,OAAO,CAAC,QAAQ;IAOhB,OAAO,CAAC,QAAQ;IAWhB,OAAO,CAAC,YAAY;IASpB,OAAO,CAAC,UAAU;IA2BlB,OAAO,CAAC,kBAAkB;IAe1B,OAAO,CAAC,iBAAiB;IAiFzB,OAAO,CAAC,YAAY;IAQpB,OAAO,CAAC,eAAe;IAQvB,OAAO,CAAC,iBAAiB;CAW1B;AAMD,wBAAgB,qBAAqB,IAAI,UAAU,CAIlD;AAED,wBAAgB,kBAAkB,IAAI;IACpC,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;CAClB,GAAG,IAAI,CAOP;AAED,wBAAsB,kBAAkB,IAAI,OAAO,CAAC,IAAI,CAAC,CAIxD"}