@j0hanz/fetch-url-mcp 1.6.0 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +507 -403
- package/dist/cli.d.ts +1 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/http/auth.d.ts +12 -0
- package/dist/http/auth.d.ts.map +1 -0
- package/dist/http/auth.js +85 -6
- package/dist/http/health.d.ts +1 -0
- package/dist/http/health.d.ts.map +1 -0
- package/dist/http/helpers.d.ts +1 -0
- package/dist/http/helpers.d.ts.map +1 -0
- package/dist/http/native.d.ts +1 -0
- package/dist/http/native.d.ts.map +1 -0
- package/dist/http/native.js +80 -63
- package/dist/http/rate-limit.d.ts +1 -0
- package/dist/http/rate-limit.d.ts.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/lib/content.d.ts +3 -0
- package/dist/lib/content.d.ts.map +1 -0
- package/dist/lib/content.js +16 -11
- package/dist/lib/core.d.ts +6 -8
- package/dist/lib/core.d.ts.map +1 -0
- package/dist/lib/core.js +111 -97
- package/dist/lib/fetch-pipeline.d.ts +1 -1
- package/dist/lib/fetch-pipeline.d.ts.map +1 -0
- package/dist/lib/fetch-pipeline.js +54 -44
- package/dist/lib/http.d.ts +1 -0
- package/dist/lib/http.d.ts.map +1 -0
- package/dist/lib/mcp-tools.d.ts +45 -7
- package/dist/lib/mcp-tools.d.ts.map +1 -0
- package/dist/lib/mcp-tools.js +37 -6
- package/dist/lib/net-utils.d.ts +1 -0
- package/dist/lib/net-utils.d.ts.map +1 -0
- package/dist/lib/progress.d.ts +1 -0
- package/dist/lib/progress.d.ts.map +1 -0
- package/dist/lib/task-handlers.d.ts +9 -1
- package/dist/lib/task-handlers.d.ts.map +1 -0
- package/dist/lib/task-handlers.js +30 -38
- package/dist/lib/types.d.ts +4 -0
- package/dist/lib/types.d.ts.map +1 -0
- package/dist/lib/types.js +12 -1
- package/dist/lib/url.d.ts +3 -0
- package/dist/lib/url.d.ts.map +1 -0
- package/dist/lib/url.js +78 -151
- package/dist/lib/utils.d.ts +2 -2
- package/dist/lib/utils.d.ts.map +1 -0
- package/dist/lib/utils.js +60 -94
- package/dist/lib/zod.d.ts +3 -0
- package/dist/lib/zod.d.ts.map +1 -0
- package/dist/lib/zod.js +33 -0
- package/dist/prompts/index.d.ts +2 -1
- package/dist/prompts/index.d.ts.map +1 -0
- package/dist/prompts/index.js +2 -13
- package/dist/resources/index.d.ts +2 -1
- package/dist/resources/index.d.ts.map +1 -0
- package/dist/resources/index.js +5 -19
- package/dist/resources/instructions.d.ts +1 -0
- package/dist/resources/instructions.d.ts.map +1 -0
- package/dist/resources/instructions.js +2 -0
- package/dist/schemas/cache.d.ts +18 -0
- package/dist/schemas/cache.d.ts.map +1 -0
- package/dist/schemas/cache.js +19 -0
- package/dist/schemas/inputs.d.ts +1 -0
- package/dist/schemas/inputs.d.ts.map +1 -0
- package/dist/schemas/outputs.d.ts +6 -5
- package/dist/schemas/outputs.d.ts.map +1 -0
- package/dist/schemas/outputs.js +5 -9
- package/dist/server.d.ts +1 -0
- package/dist/server.d.ts.map +1 -0
- package/dist/server.js +9 -7
- package/dist/tasks/execution.d.ts +1 -0
- package/dist/tasks/execution.d.ts.map +1 -0
- package/dist/tasks/execution.js +3 -21
- package/dist/tasks/manager.d.ts +2 -6
- package/dist/tasks/manager.d.ts.map +1 -0
- package/dist/tasks/manager.js +2 -4
- package/dist/tasks/owner.d.ts +1 -0
- package/dist/tasks/owner.d.ts.map +1 -0
- package/dist/tasks/tool-registry.d.ts +2 -0
- package/dist/tasks/tool-registry.d.ts.map +1 -0
- package/dist/tasks/tool-registry.js +3 -0
- package/dist/tools/fetch-url.d.ts +4 -6
- package/dist/tools/fetch-url.d.ts.map +1 -0
- package/dist/tools/fetch-url.js +61 -59
- package/dist/tools/index.d.ts +1 -0
- package/dist/tools/index.d.ts.map +1 -0
- package/dist/transform/html-translators.d.ts +1 -0
- package/dist/transform/html-translators.d.ts.map +1 -0
- package/dist/transform/html-translators.js +5 -2
- package/dist/transform/metadata.d.ts +1 -0
- package/dist/transform/metadata.d.ts.map +1 -0
- package/dist/transform/metadata.js +1 -0
- package/dist/transform/{workers/shared.d.ts → shared.d.ts} +2 -1
- package/dist/transform/shared.d.ts.map +1 -0
- package/dist/transform/{workers/shared.js → shared.js} +1 -1
- package/dist/transform/transform.d.ts +1 -0
- package/dist/transform/transform.d.ts.map +1 -0
- package/dist/transform/transform.js +21 -14
- package/dist/transform/types.d.ts +1 -4
- package/dist/transform/types.d.ts.map +1 -0
- package/dist/transform/worker-pool.d.ts +3 -18
- package/dist/transform/worker-pool.d.ts.map +1 -0
- package/dist/transform/worker-pool.js +51 -167
- package/package.json +9 -6
- package/dist/transform/workers/transform-child.d.ts +0 -1
- package/dist/transform/workers/transform-child.js +0 -15
- package/dist/transform/workers/transform-worker.d.ts +0 -1
- package/dist/transform/workers/transform-worker.js +0 -13
package/dist/tasks/manager.js
CHANGED
|
@@ -4,7 +4,7 @@ import { randomUUID } from 'node:crypto';
|
|
|
4
4
|
import { setInterval } from 'node:timers';
|
|
5
5
|
import { ErrorCode, McpError } from '@modelcontextprotocol/sdk/types.js';
|
|
6
6
|
import { config } from '../lib/core.js';
|
|
7
|
-
import {
|
|
7
|
+
import { toError } from '../lib/utils.js';
|
|
8
8
|
import { createUnrefTimeout } from '../lib/utils.js';
|
|
9
9
|
const DEFAULT_TTL_MS = 60_000;
|
|
10
10
|
const MIN_TTL_MS = 1_000;
|
|
@@ -69,8 +69,6 @@ class TaskManager {
|
|
|
69
69
|
applyTaskUpdate(task, updates) {
|
|
70
70
|
Object.assign(task, updates);
|
|
71
71
|
task.lastUpdatedAt = new Date().toISOString();
|
|
72
|
-
// Slide TTL window on every activity so long-running tasks don't expire mid-flight.
|
|
73
|
-
task._createdAtMs = Date.now();
|
|
74
72
|
}
|
|
75
73
|
cancelActiveTask(task, statusMessage) {
|
|
76
74
|
this.applyTaskUpdate(task, {
|
|
@@ -322,7 +320,7 @@ class TaskManager {
|
|
|
322
320
|
cleanup();
|
|
323
321
|
this.removeWaiter(taskId, waiter);
|
|
324
322
|
this.removeTask(taskId);
|
|
325
|
-
rejectInContext(new McpError(
|
|
323
|
+
rejectInContext(new McpError(ErrorCode.InvalidParams, 'Task expired', {
|
|
326
324
|
taskId,
|
|
327
325
|
}));
|
|
328
326
|
});
|
package/dist/tasks/owner.d.ts
CHANGED
|
@@ -30,3 +30,4 @@ export declare function resolveToolCallContext(extra?: HandlerExtra): ToolCallCo
|
|
|
30
30
|
export declare function isServerResult(value: unknown): value is ServerResult;
|
|
31
31
|
export declare function tryReadToolStructuredError(value: unknown): string | undefined;
|
|
32
32
|
export {};
|
|
33
|
+
//# sourceMappingURL=owner.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"owner.d.ts","sourceRoot":"","sources":["../../src/tasks/owner.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oCAAoC,CAAC;AAEvE,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAOhE,UAAU,YAAY;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE;QAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACjD,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC5B,gBAAgB,CAAC,EAAE,CAAC,YAAY,EAAE,oBAAoB,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1E;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC5B,gBAAgB,CAAC,EAAE,CAAC,YAAY,EAAE,oBAAoB,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1E;AAED;;sDAEsD;AACtD,KAAK,SAAS,CAAC,CAAC,SAAS,MAAM,IAAI;KAChC,CAAC,IAAI,MAAM,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,SAAS,KAAK,GACnD,KAAK,GACL,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC;CAClC,CAAC;AAEF,wBAAgB,OAAO,CAAC,CAAC,SAAS,MAAM,EAAE,GAAG,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CAQ9D;AA2BD,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,OAAO,GAAG,YAAY,GAAG,SAAS,CAyB1E;AAED,wBAAgB,mBAAmB,CAAC,KAAK,CAAC,EAAE,YAAY,GAAG,MAAM,CAMhE;AAED,wBAAgB,sBAAsB,CAAC,KAAK,CAAC,EAAE,YAAY,GAAG,eAAe,CAQ5E;AAED,wBAAgB,cAAc,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,YAAY,CAIpE;AAED,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CAkB7E"}
|
|
@@ -9,3 +9,5 @@ export declare function registerTaskCapableTool<TArgs>(descriptor: TaskCapableTo
|
|
|
9
9
|
export declare function unregisterTaskCapableTool(name: string): void;
|
|
10
10
|
export declare function getTaskCapableTool(name: string): TaskCapableToolDescriptor | undefined;
|
|
11
11
|
export declare function hasTaskCapableTool(name: string): boolean;
|
|
12
|
+
export declare function hasRegisteredTaskCapableTools(): boolean;
|
|
13
|
+
//# sourceMappingURL=tool-registry.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tool-registry.d.ts","sourceRoot":"","sources":["../../src/tasks/tool-registry.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oCAAoC,CAAC;AAEvE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAE5D,MAAM,WAAW,yBAAyB,CAAC,KAAK,GAAG,OAAO;IACxD,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,EAAE,CAAC,IAAI,EAAE,OAAO,KAAK,KAAK,CAAC;IACzC,OAAO,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,EAAE,gBAAgB,KAAK,OAAO,CAAC,YAAY,CAAC,CAAC;CAC3E;AAID,wBAAgB,uBAAuB,CAAC,KAAK,EAC3C,UAAU,EAAE,yBAAyB,CAAC,KAAK,CAAC,GAC3C,IAAI,CAKN;AAED,wBAAgB,yBAAyB,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAE5D;AAED,wBAAgB,kBAAkB,CAChC,IAAI,EAAE,MAAM,GACX,yBAAyB,GAAG,SAAS,CAEvC;AAED,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAExD;AAED,wBAAgB,6BAA6B,IAAI,OAAO,CAEvD"}
|
|
@@ -1,12 +1,9 @@
|
|
|
1
1
|
import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
|
2
2
|
import type { ContentBlock } from '@modelcontextprotocol/sdk/types.js';
|
|
3
|
+
import { z } from 'zod';
|
|
3
4
|
import { type ToolHandlerExtra } from '../lib/mcp-tools.js';
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
skipNoiseRemoval?: boolean | undefined;
|
|
7
|
-
forceRefresh?: boolean | undefined;
|
|
8
|
-
maxInlineChars?: number | undefined;
|
|
9
|
-
}
|
|
5
|
+
import { fetchUrlInputSchema } from '../schemas/inputs.js';
|
|
6
|
+
type FetchUrlInput = z.infer<typeof fetchUrlInputSchema>;
|
|
10
7
|
type ToolContentBlockUnion = ContentBlock;
|
|
11
8
|
interface ToolResponseBase {
|
|
12
9
|
[key: string]: unknown;
|
|
@@ -26,3 +23,4 @@ export declare function fetchUrlToolHandler(input: FetchUrlInput, extra?: ToolHa
|
|
|
26
23
|
export declare function withRequestContextIfMissing<TParams, TResult, TExtra = unknown>(handler: (params: TParams, extra?: TExtra) => Promise<TResult>): (params: TParams, extra?: TExtra) => Promise<TResult>;
|
|
27
24
|
export declare function registerTools(server: McpServer): void;
|
|
28
25
|
export {};
|
|
26
|
+
//# sourceMappingURL=fetch-url.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fetch-url.d.ts","sourceRoot":"","sources":["../../src/tools/fetch-url.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACV,SAAS,EAEV,MAAM,yCAAyC,CAAC;AACjD,OAAO,KAAK,EACV,YAAY,EAEb,MAAM,oCAAoC,CAAC;AAE5C,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAwBxB,OAAO,EAGL,KAAK,gBAAgB,EACtB,MAAM,qBAAqB,CAAC;AAG7B,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAS3D,KAAK,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAEzD,KAAK,qBAAqB,GAAG,YAAY,CAAC;AAE1C,UAAU,gBAAgB;IACxB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;IACvB,OAAO,EAAE,qBAAqB,EAAE,CAAC;IACjC,iBAAiB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,CAAC;IACxD,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,eAAO,MAAM,mBAAmB,cAAc,CAAC;AA8T/C,wBAAsB,mBAAmB,CACvC,KAAK,EAAE,aAAa,EACpB,KAAK,CAAC,EAAE,gBAAgB,GACvB,OAAO,CAAC,gBAAgB,CAAC,CAK3B;AAgDD;;;;;;GAMG;AACH,wBAAgB,2BAA2B,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,EAC5E,OAAO,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,GAC7D,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAmBvD;AAwBD,wBAAgB,aAAa,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI,CAwCrD"}
|
package/dist/tools/fetch-url.js
CHANGED
|
@@ -7,6 +7,7 @@ import { handleToolError } from '../lib/mcp-tools.js';
|
|
|
7
7
|
import { appendTruncationMarker, markdownTransform, parseCachedMarkdownResult, performSharedFetch, readNestedRecord, serializeMarkdownResult, TRUNCATION_MARKER, withSignal, } from '../lib/mcp-tools.js';
|
|
8
8
|
import { createProgressReporter, } from '../lib/mcp-tools.js';
|
|
9
9
|
import { isAbortError, isObject, toError } from '../lib/utils.js';
|
|
10
|
+
import { formatZodError } from '../lib/zod.js';
|
|
10
11
|
import { fetchUrlInputSchema } from '../schemas/inputs.js';
|
|
11
12
|
import { fetchUrlOutputSchema } from '../schemas/outputs.js';
|
|
12
13
|
import { registerTaskCapableTool, unregisterTaskCapableTool, } from '../tasks/tool-registry.js';
|
|
@@ -26,7 +27,6 @@ const TOOL_ICON = {
|
|
|
26
27
|
src: 'data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCAyNCAyNCIgZmlsbD0ibm9uZSIgc3Ryb2tlPSJjdXJyZW50Q29sb3IiIHN0cm9rZS13aWR0aD0iMiIgc3Ryb2tlLWxpbmVjYXA9InJvdW5kIiBzdHJva2UtbGluZWpvaW49InJvdW5kIj48cGF0aCBkPSJNMjEgMTV2NGEyIDIgMCAwIDEtMiAySDVhMiAyIDAgMCAxLTItMnYtNCIvPjxwb2x5bGluZSBwb2ludHM9IjcgMTAgMTIgMTUgMTcgMTAiLz48bGluZSB4MT0iMTIiIHkxPSIxNSIgeDI9IjEyIiB5Mj0iMyIvPjwvc3ZnPg==',
|
|
27
28
|
mimeType: 'image/svg+xml',
|
|
28
29
|
};
|
|
29
|
-
const JSON_SCHEMA_DRAFT_2020_12_URI = 'https://json-schema.org/draft/2020-12/schema';
|
|
30
30
|
/* -------------------------------------------------------------------------------------------------
|
|
31
31
|
* Tool response builders
|
|
32
32
|
* ------------------------------------------------------------------------------------------------- */
|
|
@@ -121,6 +121,42 @@ function buildResponse(pipeline, inlineResult, inputUrl) {
|
|
|
121
121
|
/* -------------------------------------------------------------------------------------------------
|
|
122
122
|
* fetch-url tool implementation
|
|
123
123
|
* ------------------------------------------------------------------------------------------------- */
|
|
124
|
+
function isCodeHost(host) {
|
|
125
|
+
return (host === 'github.com' || host === 'gitlab.com' || host === 'bitbucket.org');
|
|
126
|
+
}
|
|
127
|
+
function summarizeCodeHostPath(host, parts) {
|
|
128
|
+
if (!isCodeHost(host) || parts.length < 2)
|
|
129
|
+
return null;
|
|
130
|
+
const p0 = parts[0] ?? '';
|
|
131
|
+
const p1 = parts[1] ?? '';
|
|
132
|
+
return `${host}/${p0}/${p1}`;
|
|
133
|
+
}
|
|
134
|
+
function summarizeWikipediaPath(parts) {
|
|
135
|
+
if (parts[0] !== 'wiki' || parts.length < 2)
|
|
136
|
+
return null;
|
|
137
|
+
const p1 = parts[1] ?? '';
|
|
138
|
+
return `wikipedia.org/${p1}`;
|
|
139
|
+
}
|
|
140
|
+
function truncatePathSegment(segment, max = 20) {
|
|
141
|
+
if (segment.length <= max)
|
|
142
|
+
return segment;
|
|
143
|
+
return `${segment.substring(0, max - 3)}...`;
|
|
144
|
+
}
|
|
145
|
+
function summarizeUrlPath(host, parts) {
|
|
146
|
+
const codeHostSummary = summarizeCodeHostPath(host, parts);
|
|
147
|
+
if (codeHostSummary)
|
|
148
|
+
return codeHostSummary;
|
|
149
|
+
if (host.endsWith('wikipedia.org')) {
|
|
150
|
+
const wikipediaSummary = summarizeWikipediaPath(parts);
|
|
151
|
+
if (wikipediaSummary)
|
|
152
|
+
return wikipediaSummary;
|
|
153
|
+
}
|
|
154
|
+
const basename = truncatePathSegment(parts.at(-1) ?? '');
|
|
155
|
+
if (parts.length === 1) {
|
|
156
|
+
return `${host}/${basename}`;
|
|
157
|
+
}
|
|
158
|
+
return basename ? `${host}/…/${basename}` : host;
|
|
159
|
+
}
|
|
124
160
|
function getUrlContext(urlStr) {
|
|
125
161
|
try {
|
|
126
162
|
const u = new URL(urlStr);
|
|
@@ -131,82 +167,61 @@ function getUrlContext(urlStr) {
|
|
|
131
167
|
const parts = path.split('/').filter(Boolean);
|
|
132
168
|
if (parts.length === 0)
|
|
133
169
|
return host;
|
|
134
|
-
|
|
135
|
-
if (host === 'github.com' ||
|
|
136
|
-
host === 'gitlab.com' ||
|
|
137
|
-
host === 'bitbucket.org') {
|
|
138
|
-
if (parts.length >= 2) {
|
|
139
|
-
const p0 = parts[0] ?? '';
|
|
140
|
-
const p1 = parts[1] ?? '';
|
|
141
|
-
return `${host}/${p0}/${p1}`;
|
|
142
|
-
}
|
|
143
|
-
}
|
|
144
|
-
// Special case for Wikipedia
|
|
145
|
-
if (host.endsWith('wikipedia.org') &&
|
|
146
|
-
parts[0] === 'wiki' &&
|
|
147
|
-
parts.length >= 2) {
|
|
148
|
-
const p1 = parts[1] ?? '';
|
|
149
|
-
return `wikipedia.org/${p1}`;
|
|
150
|
-
}
|
|
151
|
-
let basename = parts.pop() ?? '';
|
|
152
|
-
if (basename && basename.length > 20) {
|
|
153
|
-
basename = `${basename.substring(0, 17)}...`;
|
|
154
|
-
}
|
|
155
|
-
if (parts.length === 0) {
|
|
156
|
-
return `${host}/${basename}`;
|
|
157
|
-
}
|
|
158
|
-
return basename ? `${host}/…/${basename}` : host;
|
|
170
|
+
return summarizeUrlPath(host, parts);
|
|
159
171
|
}
|
|
160
172
|
catch {
|
|
161
173
|
return 'unknown';
|
|
162
174
|
}
|
|
163
175
|
}
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
return performSharedFetch({
|
|
176
|
+
function buildFetchOptions(url, signal, progress, skipNoiseRemoval, forceRefresh, maxInlineChars) {
|
|
177
|
+
return {
|
|
167
178
|
url,
|
|
168
179
|
...withSignal(signal),
|
|
169
180
|
...(skipNoiseRemoval ? { cacheVary: { skipNoiseRemoval: true } } : {}),
|
|
170
181
|
...(forceRefresh ? { forceRefresh: true } : {}),
|
|
171
182
|
...(maxInlineChars !== undefined ? { maxInlineChars } : {}),
|
|
172
183
|
transform: async ({ buffer, encoding, truncated }, normalizedUrl) => {
|
|
173
|
-
|
|
184
|
+
reportProgress(progress, 2, 'Parsing HTML → Markdown');
|
|
174
185
|
return markdownTransform({ buffer, encoding, ...(truncated ? { truncated } : {}) }, normalizedUrl, signal, skipNoiseRemoval);
|
|
175
186
|
},
|
|
176
187
|
serialize: serializeMarkdownResult,
|
|
177
188
|
deserialize: parseCachedMarkdownResult,
|
|
178
|
-
}
|
|
189
|
+
};
|
|
179
190
|
}
|
|
180
|
-
function
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
191
|
+
async function fetchPipeline(url, signal, progress, skipNoiseRemoval, forceRefresh, maxInlineChars) {
|
|
192
|
+
return performSharedFetch(buildFetchOptions(url, signal, progress, skipNoiseRemoval, forceRefresh, maxInlineChars));
|
|
193
|
+
}
|
|
194
|
+
function formatContentSize(chars) {
|
|
195
|
+
if (chars < 1000)
|
|
196
|
+
return `${chars} chars`;
|
|
197
|
+
if (chars < 1_000_000)
|
|
198
|
+
return `${(chars / 1024).toFixed(1)} KB`;
|
|
199
|
+
return `${(chars / (1024 * 1024)).toFixed(1)} MB`;
|
|
185
200
|
}
|
|
186
|
-
function
|
|
201
|
+
function reportProgress(progress, step, message) {
|
|
187
202
|
if (!progress)
|
|
188
203
|
return;
|
|
189
|
-
progress.report(step,
|
|
204
|
+
progress.report(step, message);
|
|
190
205
|
}
|
|
191
206
|
async function executeFetch(input, extra) {
|
|
192
207
|
const { url } = input;
|
|
193
208
|
const signal = buildToolAbortSignal(extra?.signal);
|
|
194
209
|
const progress = createProgressReporter(extra);
|
|
195
|
-
const
|
|
196
|
-
reportFetchProgress(progress, 0, contextStr, 'starting');
|
|
210
|
+
const context = getUrlContext(url);
|
|
197
211
|
logDebug('Fetching URL', { url });
|
|
198
212
|
try {
|
|
199
|
-
|
|
213
|
+
reportProgress(progress, 1, `Fetching ${context}`);
|
|
200
214
|
const { pipeline, inlineResult } = await fetchPipeline(url, signal, progress, input.skipNoiseRemoval, input.forceRefresh, input.maxInlineChars);
|
|
201
215
|
if (pipeline.fromCache) {
|
|
202
|
-
|
|
216
|
+
reportProgress(progress, 3, 'Loaded from cache');
|
|
203
217
|
}
|
|
204
|
-
|
|
218
|
+
const size = formatContentSize(inlineResult.contentSize);
|
|
219
|
+
reportProgress(progress, 4, `Done — ${size}`);
|
|
205
220
|
return buildResponse(pipeline, inlineResult, url);
|
|
206
221
|
}
|
|
207
222
|
catch (error) {
|
|
208
223
|
const isAbort = isAbortError(error);
|
|
209
|
-
|
|
224
|
+
reportProgress(progress, 4, isAbort ? 'Cancelled' : 'Failed');
|
|
210
225
|
throw error;
|
|
211
226
|
}
|
|
212
227
|
}
|
|
@@ -216,21 +231,12 @@ export async function fetchUrlToolHandler(input, extra) {
|
|
|
216
231
|
return handleToolError(error, input.url, 'Failed to fetch URL');
|
|
217
232
|
});
|
|
218
233
|
}
|
|
219
|
-
function withJsonSchema202012(schema) {
|
|
220
|
-
if (typeof schema['$schema'] === 'string')
|
|
221
|
-
return schema;
|
|
222
|
-
return {
|
|
223
|
-
$schema: JSON_SCHEMA_DRAFT_2020_12_URI,
|
|
224
|
-
...schema,
|
|
225
|
-
};
|
|
226
|
-
}
|
|
227
234
|
const TOOL_DEFINITION = {
|
|
228
235
|
name: FETCH_URL_TOOL_NAME,
|
|
229
236
|
title: 'Fetch URL',
|
|
230
237
|
description: FETCH_URL_TOOL_DESCRIPTION,
|
|
231
238
|
inputSchema: fetchUrlInputSchema,
|
|
232
|
-
|
|
233
|
-
outputSchema: withJsonSchema202012(z.toJSONSchema(fetchUrlOutputSchema)),
|
|
239
|
+
outputSchema: z.toJSONSchema(fetchUrlOutputSchema),
|
|
234
240
|
handler: fetchUrlToolHandler,
|
|
235
241
|
execution: {
|
|
236
242
|
taskSupport: 'optional',
|
|
@@ -299,11 +305,7 @@ export function registerTools(server) {
|
|
|
299
305
|
parseArguments: (args) => {
|
|
300
306
|
const parsed = fetchUrlInputSchema.safeParse(args);
|
|
301
307
|
if (!parsed.success) {
|
|
302
|
-
|
|
303
|
-
const details = Object.entries(flat.fieldErrors)
|
|
304
|
-
.map(([k, v]) => `${k}: ${v.join(', ')}`)
|
|
305
|
-
.join('; ') || flat.formErrors.join('; ');
|
|
306
|
-
throw new McpError(ErrorCode.InvalidParams, `Invalid arguments for ${FETCH_URL_TOOL_NAME}: ${details}`);
|
|
308
|
+
throw new McpError(ErrorCode.InvalidParams, `Invalid arguments for ${FETCH_URL_TOOL_NAME}: ${formatZodError(parsed.error)}`);
|
|
307
309
|
}
|
|
308
310
|
return parsed.data;
|
|
309
311
|
},
|
package/dist/tools/index.d.ts
CHANGED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/tools/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AAIzE,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI,CAExD"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"html-translators.d.ts","sourceRoot":"","sources":["../../src/transform/html-translators.ts"],"names":[],"mappings":"AAghBA,wBAAgB,+BAA+B,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAEpE"}
|
|
@@ -157,9 +157,9 @@ function resolveImageSrc(getAttribute) {
|
|
|
157
157
|
if (url)
|
|
158
158
|
return url;
|
|
159
159
|
}
|
|
160
|
-
// If the only available src is a data URI,
|
|
160
|
+
// If the only available src is a data URI, omit the image URL entirely.
|
|
161
161
|
if (isDataUri(srcRaw))
|
|
162
|
-
return '
|
|
162
|
+
return '';
|
|
163
163
|
return '';
|
|
164
164
|
}
|
|
165
165
|
function deriveAltFromImageUrl(src) {
|
|
@@ -203,6 +203,9 @@ function buildImageTranslator(ctx) {
|
|
|
203
203
|
: undefined;
|
|
204
204
|
const src = resolveImageSrc(getAttribute);
|
|
205
205
|
const existingAlt = getAttribute?.('alt') ?? '';
|
|
206
|
+
if (!src) {
|
|
207
|
+
return { content: existingAlt.trim() };
|
|
208
|
+
}
|
|
206
209
|
const alt = existingAlt.trim() || deriveAltFromImageUrl(src);
|
|
207
210
|
const markdown = ``;
|
|
208
211
|
return { content: markdown };
|
|
@@ -2,3 +2,4 @@ import type { ExtractedMetadata } from './types.js';
|
|
|
2
2
|
export declare function extractMetadata(document: Document, baseUrl?: string): ExtractedMetadata;
|
|
3
3
|
export declare function extractMetadataFromHead(html: string, baseUrl?: string): ExtractedMetadata | null;
|
|
4
4
|
export declare function mergeMetadata(early: ExtractedMetadata | null, late: ExtractedMetadata): ExtractedMetadata;
|
|
5
|
+
//# sourceMappingURL=metadata.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"metadata.d.ts","sourceRoot":"","sources":["../../src/transform/metadata.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAyKpD,wBAAgB,eAAe,CAC7B,QAAQ,EAAE,QAAQ,EAClB,OAAO,CAAC,EAAE,MAAM,GACf,iBAAiB,CAenB;AAED,wBAAgB,uBAAuB,CACrC,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE,MAAM,GACf,iBAAiB,GAAG,IAAI,CAY1B;AAED,wBAAgB,aAAa,CAC3B,KAAK,EAAE,iBAAiB,GAAG,IAAI,EAC/B,IAAI,EAAE,iBAAiB,GACtB,iBAAiB,CAmBnB"}
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import type { MarkdownTransformResult, TransformOptions, TransformWorkerOutgoingMessage } from '
|
|
1
|
+
import type { MarkdownTransformResult, TransformOptions, TransformWorkerOutgoingMessage } from './types.js';
|
|
2
2
|
interface WorkerMessageHandlerOptions {
|
|
3
3
|
sendMessage: (message: TransformWorkerOutgoingMessage) => void;
|
|
4
4
|
runTransform: (html: string, url: string, options: TransformOptions) => MarkdownTransformResult;
|
|
5
5
|
}
|
|
6
6
|
export declare function createTransformMessageHandler(options: WorkerMessageHandlerOptions): (raw: unknown) => void;
|
|
7
7
|
export {};
|
|
8
|
+
//# sourceMappingURL=shared.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"shared.d.ts","sourceRoot":"","sources":["../../src/transform/shared.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACV,uBAAuB,EACvB,gBAAgB,EAChB,8BAA8B,EAE/B,MAAM,YAAY,CAAC;AAEpB,UAAU,2BAA2B;IACnC,WAAW,EAAE,CAAC,OAAO,EAAE,8BAA8B,KAAK,IAAI,CAAC;IAC/D,YAAY,EAAE,CACZ,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,KACtB,uBAAuB,CAAC;CAC9B;AAkGD,wBAAgB,6BAA6B,CAC3C,OAAO,EAAE,2BAA2B,GACnC,CAAC,GAAG,EAAE,OAAO,KAAK,IAAI,CAgFxB"}
|
|
@@ -34,3 +34,4 @@ type TransformExecutionOptions = TransformOptions & {
|
|
|
34
34
|
export declare function transformHtmlToMarkdown(html: string, url: string, options: TransformOptions): Promise<MarkdownTransformResult>;
|
|
35
35
|
export declare function transformBufferToMarkdown(htmlBuffer: Uint8Array, url: string, options: TransformExecutionOptions): Promise<MarkdownTransformResult>;
|
|
36
36
|
export {};
|
|
37
|
+
//# sourceMappingURL=transform.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"transform.d.ts","sourceRoot":"","sources":["../../src/transform/transform.ts"],"names":[],"mappings":"AAsCA,OAAO,KAAK,EACV,gBAAgB,EAChB,iBAAiB,EACjB,gBAAgB,EAChB,uBAAuB,EACvB,aAAa,EACb,gBAAgB,EAChB,qBAAqB,EAEtB,MAAM,YAAY,CAAC;AAqCpB,UAAU,WAAW;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;CACnB;AA4ID,wBAAgB,mBAAmB,CACjC,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,EACb,MAAM,CAAC,EAAE,WAAW,GACnB,qBAAqB,GAAG,IAAI,CAE9B;AAED,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,qBAAqB,GAAG,IAAI,EACrC,OAAO,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,OAAO,CAAA;CAAE,GAChC,MAAM,CAER;AAkUD,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,GAAE;IAAE,cAAc,CAAC,EAAE,OAAO,CAAC;IAAC,MAAM,CAAC,EAAE,WAAW,CAAA;CAExD,GACA,gBAAgB,CAGlB;AAqPD,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,QAAQ,CAAC,EAAE,aAAa,EACxB,OAAO,CAAC,EAAE;IACR,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B,GACA,MAAM,CAsBR;AAuKD,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,gBAAgB,GAAG,IAAI,EAChC,sBAAsB,EAAE,MAAM,GAAG,QAAQ,GACxC,OAAO,CAQT;AA6DD,wBAAgB,gCAAgC,CAC9C,OAAO,EAAE,gBAAgB,GAAG,IAAI,GAC/B,OAAO,IAAI,gBAAgB,CAE7B;AAED,wBAAgB,0BAA0B,CACxC,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GAAG,IAAI,EAChC,aAAa,EAAE,iBAAiB,EAChC,wBAAwB,EAAE,OAAO,EACjC,eAAe,EAAE,OAAO,GACvB,aAAa,GAAG,SAAS,CAqB3B;AAmQD,wBAAgB,gCAAgC,CAC9C,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GACxB,uBAAuB,CA8CzB;AAED,UAAU,kBAAkB;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,wBAAgB,qBAAqB,IAAI,kBAAkB,GAAG,IAAI,CAEjE;AAED,wBAAsB,2BAA2B,IAAI,OAAO,CAAC,IAAI,CAAC,CAEjE;AAED,KAAK,yBAAyB,GAAG,gBAAgB,GAAG;IAAE,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAoI1E,wBAAsB,uBAAuB,CAC3C,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,uBAAuB,CAAC,CAElC;AAED,wBAAsB,yBAAyB,CAC7C,UAAU,EAAE,UAAU,EACtB,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,yBAAyB,GACjC,OAAO,CAAC,uBAAuB,CAAC,CAElC"}
|
|
@@ -3,8 +3,7 @@ import diagnosticsChannel from 'node:diagnostics_channel';
|
|
|
3
3
|
import { performance } from 'node:perf_hooks';
|
|
4
4
|
import { isProbablyReaderable, Readability } from '@mozilla/readability';
|
|
5
5
|
import { parseHTML } from 'linkedom';
|
|
6
|
-
import { removeNoiseFromHtml } from '../lib/content.js';
|
|
7
|
-
import { addSourceToMarkdown, buildMetadataFooter, cleanupMarkdownArtifacts, extractTitleFromRawMarkdown, isRawTextContent, } from '../lib/content.js';
|
|
6
|
+
import { addSourceToMarkdown, buildMetadataFooter, cleanupMarkdownArtifacts, extractTitleFromRawMarkdown, isRawTextContent, prepareDocumentForMarkdown, removeNoiseFromHtml, serializeDocumentForMarkdown, } from '../lib/content.js';
|
|
8
7
|
import { config } from '../lib/core.js';
|
|
9
8
|
import { getOperationId, getRequestId, logDebug, logError, logInfo, logWarn, redactUrl, } from '../lib/core.js';
|
|
10
9
|
import { isRawTextContentUrl } from '../lib/http.js';
|
|
@@ -14,7 +13,6 @@ import { isObject } from '../lib/utils.js';
|
|
|
14
13
|
import { translateHtmlFragmentToMarkdown } from './html-translators.js';
|
|
15
14
|
import { extractMetadata, extractMetadataFromHead, mergeMetadata, } from './metadata.js';
|
|
16
15
|
import { getOrCreateWorkerPool, getWorkerPoolStats, shutdownWorkerPool, } from './worker-pool.js';
|
|
17
|
-
const utf8Decoder = new TextDecoder('utf-8');
|
|
18
16
|
function decodeInput(input, encoding) {
|
|
19
17
|
if (typeof input === 'string')
|
|
20
18
|
return input;
|
|
@@ -22,13 +20,15 @@ function decodeInput(input, encoding) {
|
|
|
22
20
|
if (!normalizedEncoding ||
|
|
23
21
|
normalizedEncoding === 'utf-8' ||
|
|
24
22
|
normalizedEncoding === 'utf8') {
|
|
25
|
-
|
|
23
|
+
const buf = Buffer.isBuffer(input) ? input : Buffer.from(input);
|
|
24
|
+
return buf.toString('utf8');
|
|
26
25
|
}
|
|
27
26
|
try {
|
|
28
27
|
return new TextDecoder(normalizedEncoding).decode(input);
|
|
29
28
|
}
|
|
30
29
|
catch {
|
|
31
|
-
|
|
30
|
+
const buf = Buffer.isBuffer(input) ? input : Buffer.from(input);
|
|
31
|
+
return buf.toString('utf8');
|
|
32
32
|
}
|
|
33
33
|
}
|
|
34
34
|
function asError(value) {
|
|
@@ -888,13 +888,20 @@ function buildContentSource(params) {
|
|
|
888
888
|
};
|
|
889
889
|
}
|
|
890
890
|
if (document) {
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
891
|
+
if (skipNoiseRemoval) {
|
|
892
|
+
return {
|
|
893
|
+
...base,
|
|
894
|
+
sourceHtml: html,
|
|
895
|
+
title: extractedMeta.title,
|
|
896
|
+
skipNoiseRemoval: true,
|
|
897
|
+
document,
|
|
898
|
+
};
|
|
899
|
+
}
|
|
900
|
+
prepareDocumentForMarkdown(document, url, signal);
|
|
894
901
|
const contentRoot = findContentRoot(document);
|
|
895
902
|
return {
|
|
896
903
|
...base,
|
|
897
|
-
sourceHtml: contentRoot ??
|
|
904
|
+
sourceHtml: contentRoot ?? serializeDocumentForMarkdown(document, html),
|
|
898
905
|
title: extractedMeta.title,
|
|
899
906
|
skipNoiseRemoval: true,
|
|
900
907
|
document,
|
|
@@ -912,7 +919,7 @@ function resolveContentSource(params) {
|
|
|
912
919
|
...(params.signal ? { signal: params.signal } : {}),
|
|
913
920
|
...(params.inputTruncated ? { inputTruncated: true } : {}),
|
|
914
921
|
});
|
|
915
|
-
const useArticleContent = article
|
|
922
|
+
const useArticleContent = !params.skipNoiseRemoval && article
|
|
916
923
|
? shouldUseArticleContent(article, document)
|
|
917
924
|
: false;
|
|
918
925
|
return buildContentSource({
|
|
@@ -965,11 +972,11 @@ function hasBinaryIndicators(content) {
|
|
|
965
972
|
if (content.includes('\x00'))
|
|
966
973
|
return true;
|
|
967
974
|
const sampleSize = Math.min(content.length, 2000);
|
|
968
|
-
const sample = content.slice(0, sampleSize);
|
|
969
975
|
let replacementCount = 0;
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
976
|
+
let i = -1;
|
|
977
|
+
while ((i = content.indexOf(REPLACEMENT_CHAR, i + 1)) !== -1 &&
|
|
978
|
+
i < sampleSize) {
|
|
979
|
+
replacementCount++;
|
|
973
980
|
}
|
|
974
981
|
return replacementCount > sampleSize * BINARY_INDICATOR_THRESHOLD;
|
|
975
982
|
}
|
|
@@ -101,10 +101,6 @@ export interface TransformWorkerTransformMessage {
|
|
|
101
101
|
skipNoiseRemoval?: boolean | undefined;
|
|
102
102
|
inputTruncated?: boolean | undefined;
|
|
103
103
|
}
|
|
104
|
-
export interface TransformWorkerCancelMessage {
|
|
105
|
-
type: 'cancel';
|
|
106
|
-
id: string;
|
|
107
|
-
}
|
|
108
104
|
export interface TransformWorkerCancelledMessage {
|
|
109
105
|
type: 'cancelled';
|
|
110
106
|
id: string;
|
|
@@ -127,3 +123,4 @@ export interface TransformWorkerErrorMessage {
|
|
|
127
123
|
}
|
|
128
124
|
export type TransformWorkerOutgoingMessage = TransformWorkerResultMessage | TransformWorkerErrorMessage | TransformWorkerCancelledMessage;
|
|
129
125
|
export {};
|
|
126
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/transform/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,UAAU,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,gBAAgB,GAAG,IAAI,CAAC;IACjC,QAAQ,EAAE,iBAAiB,CAAC;CAC7B;AAED,UAAU,eAAe;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,SAAS,EAAE,OAAO,CAAC;IACnB,QAAQ,CAAC,EAAE,iBAAiB,CAAC;CAC9B;AAED;;GAEG;AACH,MAAM,WAAW,uBAAwB,SAAQ,eAAe;IAC9D,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC;CAC3B;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,eAAe,EAAE,OAAO,CAAC;IACzB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,CAAC,EAAE,CAAC,CAAC;IACL,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;CACjC;AAED;;GAEG;AACH,MAAM,WAAW,+BAA+B;IAC9C,IAAI,EAAE,WAAW,CAAC;IAClB,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B,UAAU,CAAC,EAAE,UAAU,GAAG,SAAS,CAAC;IACpC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,eAAe,EAAE,OAAO,CAAC;IACzB,gBAAgB,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;IACvC,cAAc,CAAC,EAAE,OAAO,GAAG,SAAS,CAAC;CACtC;AAED,MAAM,WAAW,+BAA+B;IAC9C,IAAI,EAAE,WAAW,CAAC;IAClB,EAAE,EAAE,MAAM,CAAC;CACZ;AAED,MAAM,WAAW,4BAA4B;IAC3C,IAAI,EAAE,QAAQ,CAAC;IACf,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,eAAe,CAAC;CACzB;AAED,MAAM,WAAW,2BAA2B;IAC1C,IAAI,EAAE,OAAO,CAAC;IACd,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE;QACL,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,EAAE,MAAM,CAAC;QAChB,GAAG,EAAE,MAAM,CAAC;QACZ,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACnC,CAAC;CACH;AAED,MAAM,MAAM,8BAA8B,GACtC,4BAA4B,GAC5B,2BAA2B,GAC3B,+BAA+B,CAAC"}
|
|
@@ -1,19 +1,4 @@
|
|
|
1
|
-
import
|
|
2
|
-
import type { MarkdownTransformResult, TransformWorkerCancelMessage, TransformWorkerTransformMessage } from './types.js';
|
|
3
|
-
type TransformWorkerMessage = TransformWorkerTransformMessage | TransformWorkerCancelMessage;
|
|
4
|
-
interface WorkerHost {
|
|
5
|
-
kind: 'thread' | 'process';
|
|
6
|
-
supportsTransferList: boolean;
|
|
7
|
-
threadId?: number;
|
|
8
|
-
pid?: number;
|
|
9
|
-
postMessage: (message: TransformWorkerMessage, transferList?: NodeTransferable[]) => void;
|
|
10
|
-
terminate: () => Promise<void>;
|
|
11
|
-
unref: () => void;
|
|
12
|
-
onMessage: (handler: (raw: unknown) => void) => void;
|
|
13
|
-
onError: (handler: (error: unknown) => void) => void;
|
|
14
|
-
onExit: (handler: (code: number | null, signal: NodeJS.Signals | null) => void) => void;
|
|
15
|
-
}
|
|
16
|
-
type WorkerSpawner = (workerIndex: number, name: string) => WorkerHost;
|
|
1
|
+
import type { MarkdownTransformResult } from './types.js';
|
|
17
2
|
interface TransformWorkerPool {
|
|
18
3
|
transform(html: string, url: string, options: {
|
|
19
4
|
includeMetadata: boolean;
|
|
@@ -38,10 +23,9 @@ declare class WorkerPool implements TransformWorkerPool {
|
|
|
38
23
|
private readonly cancelAcks;
|
|
39
24
|
private readonly timeoutMs;
|
|
40
25
|
private readonly queueMax;
|
|
41
|
-
private readonly spawnWorkerImpl;
|
|
42
26
|
private closed;
|
|
43
27
|
private taskIdSeq;
|
|
44
|
-
constructor(size: number, timeoutMs: number
|
|
28
|
+
constructor(size: number, timeoutMs: number);
|
|
45
29
|
transform(html: string, url: string, options: {
|
|
46
30
|
includeMetadata: boolean;
|
|
47
31
|
signal?: AbortSignal;
|
|
@@ -90,3 +74,4 @@ export declare function getWorkerPoolStats(): {
|
|
|
90
74
|
} | null;
|
|
91
75
|
export declare function shutdownWorkerPool(): Promise<void>;
|
|
92
76
|
export {};
|
|
77
|
+
//# sourceMappingURL=worker-pool.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"worker-pool.d.ts","sourceRoot":"","sources":["../../src/transform/worker-pool.ts"],"names":[],"mappings":"AAqBA,OAAO,KAAK,EACV,uBAAuB,EAKxB,MAAM,YAAY,CAAC;AAwLpB,UAAU,mBAAmB;IAC3B,SAAS,CACP,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE;QACP,eAAe,EAAE,OAAO,CAAC;QACzB,MAAM,CAAC,EAAE,WAAW,CAAC;QACrB,gBAAgB,CAAC,EAAE,OAAO,CAAC;QAC3B,cAAc,CAAC,EAAE,OAAO,CAAC;KAC1B,GACA,OAAO,CAAC,uBAAuB,CAAC,CAAC;IACpC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACvB,aAAa,IAAI,MAAM,CAAC;IACxB,gBAAgB,IAAI,MAAM,CAAC;IAC3B,WAAW,IAAI,MAAM,CAAC;CACvB;AAkBD,cAAM,UAAW,YAAW,mBAAmB;IAC7C,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAkC;IAExE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAkC;IAC1D,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAoB;IAChD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAoB;IAEhD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAqB;IAC3C,OAAO,CAAC,SAAS,CAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAmC;IAC5D,OAAO,CAAC,QAAQ,CAAC,UAAU,CAOvB;IAEJ,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,SAAS,CAAK;gBAEV,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM;IASrC,SAAS,CACb,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE;QACP,eAAe,EAAE,OAAO,CAAC;QACzB,MAAM,CAAC,EAAE,WAAW,CAAC;QACrB,gBAAgB,CAAC,EAAE,OAAO,CAAC;QAC3B,cAAc,CAAC,EAAE,OAAO,CAAC;KAC1B,GACA,OAAO,CAAC,uBAAuB,CAAC;IAC7B,SAAS,CACb,UAAU,EAAE,UAAU,EACtB,GAAG,EAAE,MAAM,EACX,OAAO,EAAE;QACP,eAAe,EAAE,OAAO,CAAC;QACzB,MAAM,CAAC,EAAE,WAAW,CAAC;QACrB,gBAAgB,CAAC,EAAE,OAAO,CAAC;QAC3B,cAAc,CAAC,EAAE,OAAO,CAAC;QACzB,QAAQ,CAAC,EAAE,MAAM,CAAC;KACnB,GACA,OAAO,CAAC,uBAAuB,CAAC;IAoCnC,aAAa,IAAI,MAAM;IAKvB,gBAAgB,IAAI,MAAM;IAI1B,WAAW,IAAI,MAAM;IAIrB,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAWpB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAiC5B,OAAO,CAAC,UAAU;IAIlB,OAAO,CAAC,iBAAiB;IAoDzB,OAAO,CAAC,aAAa;IAsCrB,OAAO,CAAC,gBAAgB;IAOxB,OAAO,CAAC,gBAAgB;YAyBV,aAAa;IA2B3B,OAAO,CAAC,kBAAkB;IAY1B,OAAO,CAAC,WAAW;IAmCnB,OAAO,CAAC,cAAc;IAuBtB,OAAO,CAAC,aAAa;IAYrB,OAAO,CAAC,eAAe;IAuDvB,OAAO,CAAC,YAAY;IAWpB,OAAO,CAAC,QAAQ;IAOhB,OAAO,CAAC,QAAQ;IAWhB,OAAO,CAAC,YAAY;IASpB,OAAO,CAAC,UAAU;IA2BlB,OAAO,CAAC,kBAAkB;IAe1B,OAAO,CAAC,iBAAiB;IAiFzB,OAAO,CAAC,YAAY;IAQpB,OAAO,CAAC,eAAe;IAQvB,OAAO,CAAC,iBAAiB;CAW1B;AAMD,wBAAgB,qBAAqB,IAAI,UAAU,CAIlD;AAED,wBAAgB,kBAAkB,IAAI;IACpC,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;CAClB,GAAG,IAAI,CAOP;AAED,wBAAsB,kBAAkB,IAAI,OAAO,CAAC,IAAI,CAAC,CAIxD"}
|