@j0hanz/fetch-url-mcp 1.9.1 → 1.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/http/auth.d.ts +0 -1
- package/dist/http/auth.d.ts.map +1 -1
- package/dist/http/auth.js +1 -13
- package/dist/http/native.d.ts.map +1 -1
- package/dist/http/native.js +2 -5
- package/dist/lib/content.d.ts.map +1 -1
- package/dist/lib/content.js +301 -350
- package/dist/lib/core.d.ts +78 -71
- package/dist/lib/core.d.ts.map +1 -1
- package/dist/lib/core.js +308 -372
- package/dist/lib/fetch-pipeline.d.ts +2 -6
- package/dist/lib/fetch-pipeline.d.ts.map +1 -1
- package/dist/lib/fetch-pipeline.js +51 -137
- package/dist/lib/http.d.ts.map +1 -1
- package/dist/lib/http.js +188 -130
- package/dist/lib/mcp-tools.d.ts +3 -5
- package/dist/lib/mcp-tools.d.ts.map +1 -1
- package/dist/lib/mcp-tools.js +22 -58
- package/dist/lib/task-handlers.js +4 -4
- package/dist/lib/utils.d.ts +6 -0
- package/dist/lib/utils.d.ts.map +1 -1
- package/dist/lib/utils.js +23 -0
- package/dist/resources/index.js +1 -1
- package/dist/schemas.d.ts +0 -1
- package/dist/schemas.d.ts.map +1 -1
- package/dist/schemas.js +4 -6
- package/dist/server.js +1 -1
- package/dist/tasks/owner.d.ts +1 -1
- package/dist/tasks/owner.d.ts.map +1 -1
- package/dist/tasks/tool-registry.d.ts +1 -1
- package/dist/tasks/tool-registry.d.ts.map +1 -1
- package/dist/tools/fetch-url.d.ts +2 -3
- package/dist/tools/fetch-url.d.ts.map +1 -1
- package/dist/tools/fetch-url.js +89 -152
- package/dist/transform/transform.d.ts +8 -0
- package/dist/transform/transform.d.ts.map +1 -1
- package/dist/transform/transform.js +109 -108
- package/dist/transform/worker-pool.d.ts +3 -6
- package/dist/transform/worker-pool.d.ts.map +1 -1
- package/dist/transform/worker-pool.js +148 -118
- package/package.json +2 -1
|
@@ -58,23 +58,23 @@ export { cancelTasksForOwner, abortAllTaskExecutions, } from '../tasks/execution
|
|
|
58
58
|
* ------------------------------------------------------------------------------------------------- */
|
|
59
59
|
const TaskGetSchema = z.looseObject({
|
|
60
60
|
method: z.literal('tasks/get'),
|
|
61
|
-
params: z.
|
|
61
|
+
params: z.looseObject({ taskId: z.string() }),
|
|
62
62
|
});
|
|
63
63
|
const TaskListSchema = z.looseObject({
|
|
64
64
|
method: z.literal('tasks/list'),
|
|
65
65
|
params: z
|
|
66
|
-
.
|
|
66
|
+
.looseObject({
|
|
67
67
|
cursor: z.string().optional(),
|
|
68
68
|
})
|
|
69
69
|
.optional(),
|
|
70
70
|
});
|
|
71
71
|
const TaskCancelSchema = z.looseObject({
|
|
72
72
|
method: z.literal('tasks/cancel'),
|
|
73
|
-
params: z.
|
|
73
|
+
params: z.looseObject({ taskId: z.string() }),
|
|
74
74
|
});
|
|
75
75
|
const TaskResultSchema = z.looseObject({
|
|
76
76
|
method: z.literal('tasks/result'),
|
|
77
|
-
params: z.
|
|
77
|
+
params: z.looseObject({ taskId: z.string() }),
|
|
78
78
|
});
|
|
79
79
|
const MIN_TASK_TTL_MS = 1_000;
|
|
80
80
|
const MAX_TASK_TTL_MS = 86_400_000;
|
package/dist/lib/utils.d.ts
CHANGED
|
@@ -54,5 +54,11 @@ interface LikeNode {
|
|
|
54
54
|
getAttribute?(name: string): string | null;
|
|
55
55
|
}
|
|
56
56
|
export declare function isLikeNode(value: unknown): value is LikeNode;
|
|
57
|
+
type JsonRecord = Record<string, unknown>;
|
|
58
|
+
export declare function readString(obj: unknown, key: string): string | undefined;
|
|
59
|
+
export declare function readNestedRecord(obj: unknown, keys: readonly string[]): JsonRecord | undefined;
|
|
60
|
+
export declare function withSignal(signal?: AbortSignal): {
|
|
61
|
+
signal: AbortSignal;
|
|
62
|
+
} | Record<string, never>;
|
|
57
63
|
export {};
|
|
58
64
|
//# sourceMappingURL=utils.d.ts.map
|
package/dist/lib/utils.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../src/lib/utils.ts"],"names":[],"mappings":"AAwBA,wBAAgB,cAAc,CAC5B,MAAM,EAAE,WAAW,GAAG,SAAS,EAC/B,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,GACZ,IAAI,CAeN;AACD,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,UAAU,CAKvE;AAsBD,wBAAgB,mBAAmB,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,OAAO,CAcjE;AAeD,wBAAgB,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,UAAU,GAAG,MAAM,CAE5D;AACD,wBAAgB,aAAa,CAC3B,GAAG,EAAE,MAAM,GAAG,UAAU,EACxB,KAAK,EAAE,MAAM,GAAG,UAAU,GACzB,MAAM,CAER;AAED,qBAAa,UAAW,SAAQ,KAAK;IAOjC,QAAQ,CAAC,GAAG,EAAE,MAAM;IANtB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;gBAGlD,OAAO,EAAE,MAAM,EACN,GAAG,EAAE,MAAM,EACpB,UAAU,CAAC,EAAE,MAAM,EACnB,OAAO,GAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAM,EACrC,OAAO,CAAC,EAAE,YAAY;CAczB;AACD,wBAAgB,eAAe,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAsBtD;AACD,wBAAgB,OAAO,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,CAE7C;AACD,wBAAgB,YAAY,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAEpD;AACD,wBAAgB,mBAAmB,CACjC,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE,YAAY,GACrB,MAAM,CAAC,cAAc,CAGvB;AACD,wBAAgB,aAAa,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,MAAM,CAAC,cAAc,CAK5E;AAGD,wBAAgB,eAAe,CAAC,GAAG,EAAE,OAAO,GAAG,MAAM,CA8BpD;AACD,UAAU,sBAAsB;IAC9B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,IAAI,KAAK,IAAI,CAAC;IACrE,oBAAoB,CAAC,EAAE,MAAM,IAAI,CAAC;IAClC,mBAAmB,CAAC,EAAE,MAAM,IAAI,CAAC;CAClC;AAED,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,sBAAsB,GAAG,IAAI,CA2C1E;AACD,wBAAgB,0BAA0B,CACxC,MAAM,EAAE,sBAAsB,GAC7B,IAAI,CAKN;AACD,MAAM,WAAW,kBAAkB,CAAC,CAAC;IACnC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;IACpB,MAAM,EAAE,MAAM,IAAI,CAAC;CACpB;AACD,UAAU,mBAAmB,CAAC,CAAC;IAC7B,MAAM,EAAE,WAAW,CAAC;IACpB,MAAM,EAAE,CAAC,KAAK,EAAE,CAAC,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3C,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,IAAI,CAAC;CACpC;AAgBD,wBAAgB,kBAAkB,CAAC,CAAC,EAClC,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,CAAC,GACP,kBAAkB,CAAC,CAAC,CAAC,CAcvB;AACD,wBAAgB,0BAA0B,CAAC,CAAC,EAC1C,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,CAAC,EACR,OAAO,EAAE,mBAAmB,CAAC,CAAC,CAAC,GAC9B,IAAI,CAiBN;AACD,wBAAgB,QAAQ,CACtB,KAAK,EAAE,OAAO,GACb,KAAK,IAAI,MAAM,CAAC,WAAW,EAAE,OAAO,CAAC,CAEvC;AACD,wBAAgB,OAAO,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,KAAK,CAOtD;AACD,UAAU,QAAQ;IAChB,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACtC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACjD,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACxC,QAAQ,CAAC,UAAU,CAAC,EAAE,OAAO,CAAC;IAC9B,QAAQ,CAAC,UAAU,CAAC,EAAE,SAAS,CAAC,OAAO,CAAC,CAAC;IACzC,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACzC,YAAY,CAAC,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;CAC5C;AACD,wBAAgB,UAAU,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,QAAQ,CAE5D"}
|
|
1
|
+
{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../src/lib/utils.ts"],"names":[],"mappings":"AAwBA,wBAAgB,cAAc,CAC5B,MAAM,EAAE,WAAW,GAAG,SAAS,EAC/B,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,GACZ,IAAI,CAeN;AACD,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,UAAU,CAKvE;AAsBD,wBAAgB,mBAAmB,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,OAAO,CAcjE;AAeD,wBAAgB,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,UAAU,GAAG,MAAM,CAE5D;AACD,wBAAgB,aAAa,CAC3B,GAAG,EAAE,MAAM,GAAG,UAAU,EACxB,KAAK,EAAE,MAAM,GAAG,UAAU,GACzB,MAAM,CAER;AAED,qBAAa,UAAW,SAAQ,KAAK;IAOjC,QAAQ,CAAC,GAAG,EAAE,MAAM;IANtB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;gBAGlD,OAAO,EAAE,MAAM,EACN,GAAG,EAAE,MAAM,EACpB,UAAU,CAAC,EAAE,MAAM,EACnB,OAAO,GAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAM,EACrC,OAAO,CAAC,EAAE,YAAY;CAczB;AACD,wBAAgB,eAAe,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAsBtD;AACD,wBAAgB,OAAO,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,CAE7C;AACD,wBAAgB,YAAY,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAEpD;AACD,wBAAgB,mBAAmB,CACjC,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE,YAAY,GACrB,MAAM,CAAC,cAAc,CAGvB;AACD,wBAAgB,aAAa,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,MAAM,CAAC,cAAc,CAK5E;AAGD,wBAAgB,eAAe,CAAC,GAAG,EAAE,OAAO,GAAG,MAAM,CA8BpD;AACD,UAAU,sBAAsB;IAC9B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,eAAe,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,OAAO,EAAE,KAAK,IAAI,KAAK,IAAI,CAAC;IACrE,oBAAoB,CAAC,EAAE,MAAM,IAAI,CAAC;IAClC,mBAAmB,CAAC,EAAE,MAAM,IAAI,CAAC;CAClC;AAED,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,sBAAsB,GAAG,IAAI,CA2C1E;AACD,wBAAgB,0BAA0B,CACxC,MAAM,EAAE,sBAAsB,GAC7B,IAAI,CAKN;AACD,MAAM,WAAW,kBAAkB,CAAC,CAAC;IACnC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;IACpB,MAAM,EAAE,MAAM,IAAI,CAAC;CACpB;AACD,UAAU,mBAAmB,CAAC,CAAC;IAC7B,MAAM,EAAE,WAAW,CAAC;IACpB,MAAM,EAAE,CAAC,KAAK,EAAE,CAAC,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3C,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,IAAI,CAAC;CACpC;AAgBD,wBAAgB,kBAAkB,CAAC,CAAC,EAClC,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,CAAC,GACP,kBAAkB,CAAC,CAAC,CAAC,CAcvB;AACD,wBAAgB,0BAA0B,CAAC,CAAC,EAC1C,UAAU,EAAE,MAAM,EAClB,KAAK,EAAE,CAAC,EACR,OAAO,EAAE,mBAAmB,CAAC,CAAC,CAAC,GAC9B,IAAI,CAiBN;AACD,wBAAgB,QAAQ,CACtB,KAAK,EAAE,OAAO,GACb,KAAK,IAAI,MAAM,CAAC,WAAW,EAAE,OAAO,CAAC,CAEvC;AACD,wBAAgB,OAAO,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,KAAK,CAOtD;AACD,UAAU,QAAQ;IAChB,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACtC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,CAAC;IACjD,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACxC,QAAQ,CAAC,UAAU,CAAC,EAAE,OAAO,CAAC;IAC9B,QAAQ,CAAC,UAAU,CAAC,EAAE,SAAS,CAAC,OAAO,CAAC,CAAC;IACzC,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACzC,YAAY,CAAC,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;CAC5C;AACD,wBAAgB,UAAU,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,QAAQ,CAE5D;AAED,KAAK,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAQ1C,wBAAgB,UAAU,CAAC,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAGxE;AACD,wBAAgB,gBAAgB,CAC9B,GAAG,EAAE,OAAO,EACZ,IAAI,EAAE,SAAS,MAAM,EAAE,GACtB,UAAU,GAAG,SAAS,CAOxB;AACD,wBAAgB,UAAU,CACxB,MAAM,CAAC,EAAE,WAAW,GACnB;IAAE,MAAM,EAAE,WAAW,CAAA;CAAE,GAAG,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAEjD"}
|
package/dist/lib/utils.js
CHANGED
|
@@ -268,3 +268,26 @@ export function isError(value) {
|
|
|
268
268
|
export function isLikeNode(value) {
|
|
269
269
|
return isObject(value);
|
|
270
270
|
}
|
|
271
|
+
function asRecord(value) {
|
|
272
|
+
return isObject(value) ? value : undefined;
|
|
273
|
+
}
|
|
274
|
+
function readUnknown(obj, key) {
|
|
275
|
+
const record = asRecord(obj);
|
|
276
|
+
return record ? record[key] : undefined;
|
|
277
|
+
}
|
|
278
|
+
export function readString(obj, key) {
|
|
279
|
+
const value = readUnknown(obj, key);
|
|
280
|
+
return typeof value === 'string' ? value : undefined;
|
|
281
|
+
}
|
|
282
|
+
export function readNestedRecord(obj, keys) {
|
|
283
|
+
let current = obj;
|
|
284
|
+
for (const key of keys) {
|
|
285
|
+
current = readUnknown(current, key);
|
|
286
|
+
if (current === undefined)
|
|
287
|
+
return undefined;
|
|
288
|
+
}
|
|
289
|
+
return asRecord(current);
|
|
290
|
+
}
|
|
291
|
+
export function withSignal(signal) {
|
|
292
|
+
return signal === undefined ? {} : { signal };
|
|
293
|
+
}
|
package/dist/resources/index.js
CHANGED
|
@@ -2,7 +2,7 @@ import { ResourceTemplate, } from '@modelcontextprotocol/sdk/server/mcp.js';
|
|
|
2
2
|
import { ErrorCode, McpError, SubscribeRequestSchema, UnsubscribeRequestSchema, } from '@modelcontextprotocol/sdk/types.js';
|
|
3
3
|
import { get as getCacheEntry, getEntryMeta, keys as listCacheKeys, onCacheUpdate, parseCacheKey, } from '../lib/core.js';
|
|
4
4
|
import { logWarn } from '../lib/core.js';
|
|
5
|
-
import { registerServerLifecycleCleanup } from '../lib/
|
|
5
|
+
import { registerServerLifecycleCleanup } from '../lib/task-handlers.js';
|
|
6
6
|
import { buildOptionalIcons } from '../lib/types.js';
|
|
7
7
|
import { isObject } from '../lib/utils.js';
|
|
8
8
|
import { parseCachedPayload, resolveCachedPayloadContent } from '../schemas.js';
|
package/dist/schemas.d.ts
CHANGED
|
@@ -22,7 +22,6 @@ export declare function normalizeExtractedMetadata(value: unknown): ExtractedMet
|
|
|
22
22
|
export declare function normalizePageTitle(value: unknown): string | undefined;
|
|
23
23
|
export declare const cachedPayloadSchema: z.ZodObject<{
|
|
24
24
|
markdown: z.ZodOptional<z.ZodPipe<z.ZodUnknown, z.ZodTransform<string | undefined, unknown>>>;
|
|
25
|
-
content: z.ZodOptional<z.ZodPipe<z.ZodUnknown, z.ZodTransform<string | undefined, unknown>>>;
|
|
26
25
|
title: z.ZodOptional<z.ZodPipe<z.ZodUnknown, z.ZodTransform<string | undefined, unknown>>>;
|
|
27
26
|
metadata: z.ZodOptional<z.ZodPipe<z.ZodUnknown, z.ZodTransform<ExtractedMetadata | undefined, unknown>>>;
|
|
28
27
|
truncated: z.ZodOptional<z.ZodPipe<z.ZodUnknown, z.ZodTransform<boolean | undefined, unknown>>>;
|
package/dist/schemas.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schemas.d.ts","sourceRoot":"","sources":["../src/schemas.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAI9D,eAAO,MAAM,eAAe;;;;;;;;CAQlB,CAAC;AAiCX,eAAO,MAAM,uBAAuB;;;;;;;;kBAQlC,CAAC;AAgBH,wBAAgB,0BAA0B,CACxC,KAAK,EAAE,OAAO,GACb,iBAAiB,GAAG,SAAS,CAQ/B;AAED,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CAErE;AAUD,eAAO,MAAM,mBAAmB
|
|
1
|
+
{"version":3,"file":"schemas.d.ts","sourceRoot":"","sources":["../src/schemas.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAI9D,eAAO,MAAM,eAAe;;;;;;;;CAQlB,CAAC;AAiCX,eAAO,MAAM,uBAAuB;;;;;;;;kBAQlC,CAAC;AAgBH,wBAAgB,0BAA0B,CACxC,KAAK,EAAE,OAAO,GACb,iBAAiB,GAAG,SAAS,CAQ/B;AAED,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CAErE;AAUD,eAAO,MAAM,mBAAmB;;;;;iBAqB5B,CAAC;AAEL,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAEhE,eAAO,MAAM,mBAAmB;;;kBAU9B,CAAC;AAEH,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;;kBAqC/B,CAAC;AAEH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,GAAG,aAAa,GAAG,IAAI,CAkBpE;AAED,wBAAgB,2BAA2B,CACzC,OAAO,EAAE,aAAa,GACrB,MAAM,GAAG,IAAI,CAEf"}
|
package/dist/schemas.js
CHANGED
|
@@ -71,10 +71,6 @@ export const cachedPayloadSchema = z
|
|
|
71
71
|
.unknown()
|
|
72
72
|
.transform((value) => normalizeString(value))
|
|
73
73
|
.optional(),
|
|
74
|
-
content: z
|
|
75
|
-
.unknown()
|
|
76
|
-
.transform((value) => normalizeString(value))
|
|
77
|
-
.optional(),
|
|
78
74
|
title: z
|
|
79
75
|
.unknown()
|
|
80
76
|
.transform((value) => normalizePageTitle(value))
|
|
@@ -88,7 +84,9 @@ export const cachedPayloadSchema = z
|
|
|
88
84
|
.transform((value) => normalizeBoolean(value))
|
|
89
85
|
.optional(),
|
|
90
86
|
})
|
|
91
|
-
.refine((value) => typeof value.markdown === 'string'
|
|
87
|
+
.refine((value) => typeof value.markdown === 'string', {
|
|
88
|
+
error: 'Missing markdown',
|
|
89
|
+
});
|
|
92
90
|
export const fetchUrlInputSchema = z.strictObject({
|
|
93
91
|
url: z
|
|
94
92
|
.httpUrl()
|
|
@@ -158,5 +156,5 @@ export function parseCachedPayload(raw) {
|
|
|
158
156
|
}
|
|
159
157
|
}
|
|
160
158
|
export function resolveCachedPayloadContent(payload) {
|
|
161
|
-
return payload.markdown ??
|
|
159
|
+
return payload.markdown ?? null;
|
|
162
160
|
}
|
package/dist/server.js
CHANGED
|
@@ -5,7 +5,7 @@ import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'
|
|
|
5
5
|
import { SetLevelRequestSchema } from '@modelcontextprotocol/sdk/types.js';
|
|
6
6
|
import { config } from './lib/core.js';
|
|
7
7
|
import { getSessionId, logError, logInfo, setLogLevel, setMcpServer, } from './lib/core.js';
|
|
8
|
-
import { abortAllTaskExecutions, registerTaskHandlers, } from './lib/
|
|
8
|
+
import { abortAllTaskExecutions, registerTaskHandlers, } from './lib/task-handlers.js';
|
|
9
9
|
import { toError } from './lib/utils.js';
|
|
10
10
|
import { registerGetHelpPrompt } from './prompts/index.js';
|
|
11
11
|
import { registerCacheResourceTemplate, registerInstructionResource, } from './resources/index.js';
|
package/dist/tasks/owner.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"owner.d.ts","sourceRoot":"","sources":["../../src/tasks/owner.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oCAAoC,CAAC;AAGvE,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"owner.d.ts","sourceRoot":"","sources":["../../src/tasks/owner.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oCAAoC,CAAC;AAGvE,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAO/D,UAAU,YAAY;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE;QAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACjD,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC5B,gBAAgB,CAAC,EAAE,CAAC,YAAY,EAAE,oBAAoB,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1E;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC5B,gBAAgB,CAAC,EAAE,CAAC,YAAY,EAAE,oBAAoB,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1E;AAED;;sDAEsD;AACtD,KAAK,SAAS,CAAC,CAAC,SAAS,MAAM,IAAI;KAChC,CAAC,IAAI,MAAM,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,SAAS,KAAK,GACnD,KAAK,GACL,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC;CAClC,CAAC;AAEF,wBAAgB,OAAO,CAAC,CAAC,SAAS,MAAM,EAAE,GAAG,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CAQ9D;AA2BD,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,OAAO,GAAG,YAAY,GAAG,SAAS,CAyB1E;AAED,wBAAgB,mBAAmB,CAAC,KAAK,CAAC,EAAE,YAAY,GAAG,MAAM,CAMhE;AAED,wBAAgB,sBAAsB,CAAC,KAAK,CAAC,EAAE,YAAY,GAAG,eAAe,CAQ5E;AAED,wBAAgB,cAAc,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,YAAY,CAIpE;AAWD,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CAgB7E"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { ServerResult } from '@modelcontextprotocol/sdk/types.js';
|
|
2
|
-
import type { ToolHandlerExtra } from '../lib/
|
|
2
|
+
import type { ToolHandlerExtra } from '../lib/progress.js';
|
|
3
3
|
export interface TaskCapableToolDescriptor<TArgs = unknown> {
|
|
4
4
|
name: string;
|
|
5
5
|
parseArguments: (args: unknown) => TArgs;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tool-registry.d.ts","sourceRoot":"","sources":["../../src/tasks/tool-registry.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oCAAoC,CAAC;AAEvE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"tool-registry.d.ts","sourceRoot":"","sources":["../../src/tasks/tool-registry.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oCAAoC,CAAC;AAEvE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAE3D,MAAM,WAAW,yBAAyB,CAAC,KAAK,GAAG,OAAO;IACxD,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,EAAE,CAAC,IAAI,EAAE,OAAO,KAAK,KAAK,CAAC;IACzC,OAAO,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,EAAE,gBAAgB,KAAK,OAAO,CAAC,YAAY,CAAC,CAAC;CAC3E;AAID,wBAAgB,uBAAuB,CAAC,KAAK,EAC3C,UAAU,EAAE,yBAAyB,CAAC,KAAK,CAAC,GAC3C,IAAI,CAKN;AAED,wBAAgB,yBAAyB,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAE5D;AAED,wBAAgB,kBAAkB,CAChC,IAAI,EAAE,MAAM,GACX,yBAAyB,GAAG,SAAS,CAEvC;AAED,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAExD;AAED,wBAAgB,6BAA6B,IAAI,OAAO,CAEvD"}
|
|
@@ -1,13 +1,12 @@
|
|
|
1
1
|
import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
|
2
2
|
import type { ContentBlock } from '@modelcontextprotocol/sdk/types.js';
|
|
3
3
|
import { z } from 'zod';
|
|
4
|
-
import { type ToolHandlerExtra } from '../lib/
|
|
4
|
+
import { type ToolHandlerExtra } from '../lib/progress.js';
|
|
5
5
|
import { fetchUrlInputSchema } from '../schemas.js';
|
|
6
6
|
type FetchUrlInput = z.infer<typeof fetchUrlInputSchema>;
|
|
7
|
-
type ToolContentBlockUnion = ContentBlock;
|
|
8
7
|
interface ToolResponseBase {
|
|
9
8
|
[key: string]: unknown;
|
|
10
|
-
content:
|
|
9
|
+
content: ContentBlock[];
|
|
11
10
|
structuredContent?: Record<string, unknown> | undefined;
|
|
12
11
|
isError?: boolean;
|
|
13
12
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fetch-url.d.ts","sourceRoot":"","sources":["../../src/tools/fetch-url.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,
|
|
1
|
+
{"version":3,"file":"fetch-url.d.ts","sourceRoot":"","sources":["../../src/tools/fetch-url.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AACzE,OAAO,KAAK,EACV,YAAY,EAEb,MAAM,oCAAoC,CAAC;AAE5C,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAyBxB,OAAO,EAGL,KAAK,gBAAgB,EACtB,MAAM,oBAAoB,CAAC;AAI5B,OAAO,EACL,mBAAmB,EAIpB,MAAM,eAAe,CAAC;AAMvB,KAAK,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAEzD,UAAU,gBAAgB;IACxB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;IACvB,OAAO,EAAE,YAAY,EAAE,CAAC;IACxB,iBAAiB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,CAAC;IACxD,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,eAAO,MAAM,mBAAmB,cAAc,CAAC;AAuN/C,wBAAsB,mBAAmB,CACvC,KAAK,EAAE,aAAa,EACpB,KAAK,CAAC,EAAE,gBAAgB,GACvB,OAAO,CAAC,gBAAgB,CAAC,CAK3B;AAsBD;;;;;;GAMG;AACH,wBAAgB,2BAA2B,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,EAC5E,OAAO,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,GAC7D,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAmBvD;AAwBD,wBAAgB,aAAa,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI,CAqCrD"}
|
package/dist/tools/fetch-url.js
CHANGED
|
@@ -1,11 +1,10 @@
|
|
|
1
1
|
import { randomUUID } from 'node:crypto';
|
|
2
2
|
import { ErrorCode, McpError } from '@modelcontextprotocol/sdk/types.js';
|
|
3
3
|
import { z } from 'zod';
|
|
4
|
-
import { config } from '../lib/core.js';
|
|
5
|
-
import {
|
|
4
|
+
import { config, getRequestId, logDebug, logError, logWarn, runWithRequestContext, } from '../lib/core.js';
|
|
5
|
+
import { appendTruncationMarker, markdownTransform, parseCachedMarkdownResult, performSharedFetch, readNestedRecord, serializeMarkdownResult, TRUNCATION_MARKER, withSignal, } from '../lib/fetch-pipeline.js';
|
|
6
6
|
import { handleToolError } from '../lib/mcp-tools.js';
|
|
7
|
-
import {
|
|
8
|
-
import { createProgressReporter, } from '../lib/mcp-tools.js';
|
|
7
|
+
import { createProgressReporter, } from '../lib/progress.js';
|
|
9
8
|
import { isAbortError, isObject, toError } from '../lib/utils.js';
|
|
10
9
|
import { formatZodError } from '../lib/zod.js';
|
|
11
10
|
import { fetchUrlInputSchema, fetchUrlOutputSchema, normalizeExtractedMetadata, normalizePageTitle, } from '../schemas.js';
|
|
@@ -26,40 +25,71 @@ const TOOL_ICON = {
|
|
|
26
25
|
src: 'data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgMCAyNCAyNCIgZmlsbD0ibm9uZSIgc3Ryb2tlPSJjdXJyZW50Q29sb3IiIHN0cm9rZS13aWR0aD0iMiIgc3Ryb2tlLWxpbmVjYXA9InJvdW5kIiBzdHJva2UtbGluZWpvaW49InJvdW5kIj48cGF0aCBkPSJNMjEgMTV2NGEyIDIgMCAwIDEtMiAySDVhMiAyIDAgMCAxLTItMnYtNCIvPjxwb2x5bGluZSBwb2ludHM9IjcgMTAgMTIgMTUgMTcgMTAiLz48bGluZSB4MT0iMTIiIHkxPSIxNSIgeDI9IjEyIiB5Mj0iMyIvPjwvc3ZnPg==',
|
|
27
26
|
mimeType: 'image/svg+xml',
|
|
28
27
|
};
|
|
28
|
+
const HARD_TOOL_TIMEOUT_MS = 300_000;
|
|
29
|
+
const CODE_HOSTS = new Set(['github.com', 'gitlab.com', 'bitbucket.org']);
|
|
29
30
|
/* -------------------------------------------------------------------------------------------------
|
|
30
|
-
*
|
|
31
|
+
* URL context & progress
|
|
31
32
|
* ------------------------------------------------------------------------------------------------- */
|
|
32
|
-
function
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
33
|
+
function getUrlContext(urlStr) {
|
|
34
|
+
try {
|
|
35
|
+
const u = new URL(urlStr);
|
|
36
|
+
const host = u.hostname.replace(/^www\./, '');
|
|
37
|
+
const parts = u.pathname.split('/').filter(Boolean);
|
|
38
|
+
if (parts.length === 0)
|
|
39
|
+
return host;
|
|
40
|
+
if (CODE_HOSTS.has(host) && parts.length >= 2) {
|
|
41
|
+
return `${host}/${parts[0] ?? ''}/${parts[1] ?? ''}`;
|
|
42
|
+
}
|
|
43
|
+
if (host.endsWith('wikipedia.org') &&
|
|
44
|
+
parts[0] === 'wiki' &&
|
|
45
|
+
parts.length >= 2) {
|
|
46
|
+
return `wikipedia.org/${parts[1] ?? ''}`;
|
|
47
|
+
}
|
|
48
|
+
const raw = parts.at(-1) ?? '';
|
|
49
|
+
const basename = raw.length > 20 ? `${raw.substring(0, 17)}...` : raw;
|
|
50
|
+
if (parts.length === 1)
|
|
51
|
+
return `${host}/${basename}`;
|
|
52
|
+
return basename ? `${host}/…/${basename}` : host;
|
|
53
|
+
}
|
|
54
|
+
catch {
|
|
55
|
+
return 'unknown';
|
|
56
|
+
}
|
|
37
57
|
}
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
58
|
+
function mapFetchStageToProgress(stage, context) {
|
|
59
|
+
switch (stage) {
|
|
60
|
+
case 'resolve_url':
|
|
61
|
+
return { step: 2, message: 'Resolving URL' };
|
|
62
|
+
case 'check_cache':
|
|
63
|
+
return { step: 3, message: 'Checking cache' };
|
|
64
|
+
case 'cache_hit':
|
|
65
|
+
return { step: 4, message: 'Loaded from cache' };
|
|
66
|
+
case 'cache_restore':
|
|
67
|
+
return { step: 5, message: 'Restoring cached content' };
|
|
68
|
+
case 'fetch_remote':
|
|
69
|
+
return { step: 4, message: `Fetching ${context}` };
|
|
70
|
+
case 'response_ready':
|
|
71
|
+
return { step: 5, message: 'Received response' };
|
|
72
|
+
case 'transform_start':
|
|
73
|
+
return { step: 6, message: 'Parsing HTML → Markdown' };
|
|
74
|
+
case 'prepare_output':
|
|
75
|
+
return { step: 6, message: 'Preparing output' };
|
|
76
|
+
case 'finalize_output':
|
|
77
|
+
return { step: 7, message: 'Finalizing output' };
|
|
78
|
+
}
|
|
49
79
|
}
|
|
50
80
|
/* -------------------------------------------------------------------------------------------------
|
|
51
|
-
*
|
|
81
|
+
* Response assembly
|
|
52
82
|
* ------------------------------------------------------------------------------------------------- */
|
|
53
|
-
function truncateStr(value, max) {
|
|
54
|
-
if (value === undefined || value.length <= max)
|
|
55
|
-
return value;
|
|
56
|
-
return value.slice(0, max);
|
|
57
|
-
}
|
|
58
83
|
function buildStructuredContent(pipeline, inlineResult, inputUrl) {
|
|
59
84
|
const truncated = inlineResult.truncated ?? pipeline.data.truncated;
|
|
60
|
-
|
|
85
|
+
let markdown = inlineResult.content;
|
|
86
|
+
if (pipeline.data.truncated && typeof markdown === 'string') {
|
|
87
|
+
markdown = appendTruncationMarker(markdown, TRUNCATION_MARKER);
|
|
88
|
+
}
|
|
61
89
|
const maxChars = config.constants.maxInlineContentChars;
|
|
62
|
-
|
|
90
|
+
if (maxChars > 0 && markdown !== undefined && markdown.length > maxChars) {
|
|
91
|
+
markdown = markdown.slice(0, maxChars);
|
|
92
|
+
}
|
|
63
93
|
const metadata = normalizeExtractedMetadata(pipeline.data.metadata);
|
|
64
94
|
const title = normalizePageTitle(pipeline.data.title);
|
|
65
95
|
return {
|
|
@@ -76,108 +106,30 @@ function buildStructuredContent(pipeline, inlineResult, inputUrl) {
|
|
|
76
106
|
...(truncated ? { truncated: true } : {}),
|
|
77
107
|
};
|
|
78
108
|
}
|
|
79
|
-
function applyTruncationMarker(content, truncated) {
|
|
80
|
-
if (!truncated || typeof content !== 'string')
|
|
81
|
-
return content;
|
|
82
|
-
return appendTruncationMarker(content, TRUNCATION_MARKER);
|
|
83
|
-
}
|
|
84
|
-
function buildFetchUrlContentBlocks(structuredContent) {
|
|
85
|
-
return [buildTextBlock(structuredContent)];
|
|
86
|
-
}
|
|
87
109
|
function buildResponse(pipeline, inlineResult, inputUrl) {
|
|
88
110
|
const structuredContent = buildStructuredContent(pipeline, inlineResult, inputUrl);
|
|
89
|
-
const content =
|
|
111
|
+
const content = [
|
|
112
|
+
{ type: 'text', text: JSON.stringify(structuredContent) },
|
|
113
|
+
];
|
|
90
114
|
const validation = fetchUrlOutputSchema.safeParse(structuredContent);
|
|
91
115
|
if (!validation.success) {
|
|
92
116
|
logWarn('Tool output schema validation failed', {
|
|
93
117
|
url: inputUrl,
|
|
94
118
|
issues: formatZodError(validation.error),
|
|
95
119
|
});
|
|
96
|
-
// Omit structuredContent so the SDK does not receive data that fails its
|
|
97
|
-
// output schema validation. The client still gets the payload via content[0].text.
|
|
98
120
|
return { content };
|
|
99
121
|
}
|
|
100
|
-
return {
|
|
101
|
-
content,
|
|
102
|
-
structuredContent,
|
|
103
|
-
};
|
|
122
|
+
return { content, structuredContent };
|
|
104
123
|
}
|
|
105
124
|
/* -------------------------------------------------------------------------------------------------
|
|
106
|
-
*
|
|
125
|
+
* Fetch pipeline
|
|
107
126
|
* ------------------------------------------------------------------------------------------------- */
|
|
108
|
-
function
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
const p0 = parts[0] ?? '';
|
|
115
|
-
const p1 = parts[1] ?? '';
|
|
116
|
-
return `${host}/${p0}/${p1}`;
|
|
117
|
-
}
|
|
118
|
-
function summarizeWikipediaPath(parts) {
|
|
119
|
-
if (parts[0] !== 'wiki' || parts.length < 2)
|
|
120
|
-
return null;
|
|
121
|
-
const p1 = parts[1] ?? '';
|
|
122
|
-
return `wikipedia.org/${p1}`;
|
|
123
|
-
}
|
|
124
|
-
function truncatePathSegment(segment, max = 20) {
|
|
125
|
-
if (segment.length <= max)
|
|
126
|
-
return segment;
|
|
127
|
-
return `${segment.substring(0, max - 3)}...`;
|
|
128
|
-
}
|
|
129
|
-
function summarizeUrlPath(host, parts) {
|
|
130
|
-
const codeHostSummary = summarizeCodeHostPath(host, parts);
|
|
131
|
-
if (codeHostSummary)
|
|
132
|
-
return codeHostSummary;
|
|
133
|
-
if (host.endsWith('wikipedia.org')) {
|
|
134
|
-
const wikipediaSummary = summarizeWikipediaPath(parts);
|
|
135
|
-
if (wikipediaSummary)
|
|
136
|
-
return wikipediaSummary;
|
|
137
|
-
}
|
|
138
|
-
const basename = truncatePathSegment(parts.at(-1) ?? '');
|
|
139
|
-
if (parts.length === 1) {
|
|
140
|
-
return `${host}/${basename}`;
|
|
141
|
-
}
|
|
142
|
-
return basename ? `${host}/…/${basename}` : host;
|
|
143
|
-
}
|
|
144
|
-
function getUrlContext(urlStr) {
|
|
145
|
-
try {
|
|
146
|
-
const u = new URL(urlStr);
|
|
147
|
-
const host = u.hostname.replace(/^www\./, '');
|
|
148
|
-
const path = u.pathname;
|
|
149
|
-
if (path === '/' || path === '')
|
|
150
|
-
return host;
|
|
151
|
-
const parts = path.split('/').filter(Boolean);
|
|
152
|
-
if (parts.length === 0)
|
|
153
|
-
return host;
|
|
154
|
-
return summarizeUrlPath(host, parts);
|
|
155
|
-
}
|
|
156
|
-
catch {
|
|
157
|
-
return 'unknown';
|
|
158
|
-
}
|
|
159
|
-
}
|
|
160
|
-
function mapFetchStageToProgress(stage, context) {
|
|
161
|
-
switch (stage) {
|
|
162
|
-
case 'resolve_url':
|
|
163
|
-
return { step: 2, message: 'Resolving URL' };
|
|
164
|
-
case 'check_cache':
|
|
165
|
-
return { step: 3, message: 'Checking cache' };
|
|
166
|
-
case 'cache_hit':
|
|
167
|
-
return { step: 4, message: 'Loaded from cache' };
|
|
168
|
-
case 'cache_restore':
|
|
169
|
-
return { step: 5, message: 'Restoring cached content' };
|
|
170
|
-
case 'fetch_remote':
|
|
171
|
-
return { step: 4, message: `Fetching ${context}` };
|
|
172
|
-
case 'response_ready':
|
|
173
|
-
return { step: 5, message: 'Received response' };
|
|
174
|
-
case 'transform_start':
|
|
175
|
-
return { step: 6, message: 'Parsing HTML → Markdown' };
|
|
176
|
-
case 'prepare_output':
|
|
177
|
-
return { step: 6, message: 'Preparing output' };
|
|
178
|
-
case 'finalize_output':
|
|
179
|
-
return { step: 7, message: 'Finalizing output' };
|
|
180
|
-
}
|
|
127
|
+
function buildToolAbortSignal(extraSignal) {
|
|
128
|
+
const timeout = config.tools.timeoutMs > 0 ? config.tools.timeoutMs : HARD_TOOL_TIMEOUT_MS;
|
|
129
|
+
const timeoutSignal = AbortSignal.timeout(timeout);
|
|
130
|
+
return extraSignal
|
|
131
|
+
? AbortSignal.any([extraSignal, timeoutSignal])
|
|
132
|
+
: timeoutSignal;
|
|
181
133
|
}
|
|
182
134
|
function buildFetchOptions(url, context, signal, progress, forceRefresh) {
|
|
183
135
|
return {
|
|
@@ -185,8 +137,8 @@ function buildFetchOptions(url, context, signal, progress, forceRefresh) {
|
|
|
185
137
|
...withSignal(signal),
|
|
186
138
|
...(forceRefresh ? { forceRefresh: true } : {}),
|
|
187
139
|
onStage: (stage) => {
|
|
188
|
-
const
|
|
189
|
-
|
|
140
|
+
const { step, message } = mapFetchStageToProgress(stage, context);
|
|
141
|
+
progress?.report(step, message);
|
|
190
142
|
},
|
|
191
143
|
transform: async ({ buffer, encoding, truncated }, normalizedUrl) => {
|
|
192
144
|
return markdownTransform({ buffer, encoding, ...(truncated ? { truncated } : {}) }, normalizedUrl, signal);
|
|
@@ -195,21 +147,6 @@ function buildFetchOptions(url, context, signal, progress, forceRefresh) {
|
|
|
195
147
|
deserialize: parseCachedMarkdownResult,
|
|
196
148
|
};
|
|
197
149
|
}
|
|
198
|
-
async function fetchPipeline(url, context, signal, progress, forceRefresh) {
|
|
199
|
-
return performSharedFetch(buildFetchOptions(url, context, signal, progress, forceRefresh));
|
|
200
|
-
}
|
|
201
|
-
function formatContentSize(chars) {
|
|
202
|
-
if (chars < 1000)
|
|
203
|
-
return `${chars} chars`;
|
|
204
|
-
if (chars < 1_000_000)
|
|
205
|
-
return `${(chars / 1024).toFixed(1)} KB`;
|
|
206
|
-
return `${(chars / (1024 * 1024)).toFixed(1)} MB`;
|
|
207
|
-
}
|
|
208
|
-
function reportProgress(progress, step, message) {
|
|
209
|
-
if (!progress)
|
|
210
|
-
return;
|
|
211
|
-
progress.report(step, message);
|
|
212
|
-
}
|
|
213
150
|
async function executeFetch(input, extra) {
|
|
214
151
|
const { url } = input;
|
|
215
152
|
const signal = buildToolAbortSignal(extra?.signal);
|
|
@@ -217,15 +154,19 @@ async function executeFetch(input, extra) {
|
|
|
217
154
|
const context = getUrlContext(url);
|
|
218
155
|
logDebug('Fetching URL', { url });
|
|
219
156
|
try {
|
|
220
|
-
|
|
221
|
-
const { pipeline, inlineResult } = await
|
|
222
|
-
const
|
|
223
|
-
|
|
157
|
+
progress.report(1, 'Preparing request');
|
|
158
|
+
const { pipeline, inlineResult } = await performSharedFetch(buildFetchOptions(url, context, signal, progress, input.forceRefresh));
|
|
159
|
+
const chars = inlineResult.contentSize;
|
|
160
|
+
const size = chars < 1000
|
|
161
|
+
? `${chars} chars`
|
|
162
|
+
: chars < 1_000_000
|
|
163
|
+
? `${(chars / 1024).toFixed(1)} KB`
|
|
164
|
+
: `${(chars / (1024 * 1024)).toFixed(1)} MB`;
|
|
165
|
+
progress.report(8, `Done — ${size}`);
|
|
224
166
|
return buildResponse(pipeline, inlineResult, url);
|
|
225
167
|
}
|
|
226
168
|
catch (error) {
|
|
227
|
-
|
|
228
|
-
reportProgress(progress, 8, isAbort ? 'Cancelled' : 'Failed');
|
|
169
|
+
progress.report(8, isAbortError(error) ? 'Cancelled' : 'Failed');
|
|
229
170
|
throw error;
|
|
230
171
|
}
|
|
231
172
|
}
|
|
@@ -235,6 +176,9 @@ export async function fetchUrlToolHandler(input, extra) {
|
|
|
235
176
|
return handleToolError(error, input.url, 'Failed to fetch URL');
|
|
236
177
|
});
|
|
237
178
|
}
|
|
179
|
+
/* -------------------------------------------------------------------------------------------------
|
|
180
|
+
* MCP tool definition + registration
|
|
181
|
+
* ------------------------------------------------------------------------------------------------- */
|
|
238
182
|
const TOOL_DEFINITION = {
|
|
239
183
|
name: FETCH_URL_TOOL_NAME,
|
|
240
184
|
title: 'Fetch URL',
|
|
@@ -242,9 +186,7 @@ const TOOL_DEFINITION = {
|
|
|
242
186
|
inputSchema: fetchUrlInputSchema,
|
|
243
187
|
outputSchema: z.toJSONSchema(fetchUrlOutputSchema),
|
|
244
188
|
handler: fetchUrlToolHandler,
|
|
245
|
-
execution: {
|
|
246
|
-
taskSupport: 'optional',
|
|
247
|
-
},
|
|
189
|
+
execution: { taskSupport: 'optional' },
|
|
248
190
|
annotations: {
|
|
249
191
|
readOnlyHint: true,
|
|
250
192
|
destructiveHint: false,
|
|
@@ -252,11 +194,6 @@ const TOOL_DEFINITION = {
|
|
|
252
194
|
openWorldHint: true,
|
|
253
195
|
},
|
|
254
196
|
};
|
|
255
|
-
function applyRegisteredToolExecutionMetadata(registeredTool, execution) {
|
|
256
|
-
// SDK workaround: RegisteredTool does not expose `execution` in its public type.
|
|
257
|
-
// Keep the mutation localized to one helper so future SDK upgrades touch one place.
|
|
258
|
-
registeredTool.execution = execution;
|
|
259
|
-
}
|
|
260
197
|
/**
|
|
261
198
|
* Stdio-path guard: ensures a request context (requestId, sessionId) is set
|
|
262
199
|
* in AsyncLocalStorage before invoking the handler. On the HTTP path the SDK
|
|
@@ -324,7 +261,7 @@ export function registerTools(server) {
|
|
|
324
261
|
execution: TOOL_DEFINITION.execution,
|
|
325
262
|
icons: [TOOL_ICON],
|
|
326
263
|
}, withRequestContextIfMissing(TOOL_DEFINITION.handler));
|
|
327
|
-
// SDK
|
|
328
|
-
|
|
329
|
-
|
|
264
|
+
// SDK workaround: RegisteredTool type omits `execution`
|
|
265
|
+
registeredTool.execution =
|
|
266
|
+
TOOL_DEFINITION.execution;
|
|
330
267
|
}
|
|
@@ -20,6 +20,14 @@ export declare function htmlToMarkdown(html: string, metadata?: MetadataBlock, o
|
|
|
20
20
|
export declare function isExtractionSufficient(article: ExtractedArticle | null, originalHtmlOrDocument: string | Document): boolean;
|
|
21
21
|
export declare function determineContentExtractionSource(article: ExtractedArticle | null): article is ExtractedArticle;
|
|
22
22
|
export declare function createContentMetadataBlock(url: string, article: ExtractedArticle | null, extractedMeta: ExtractedMetadata, shouldExtractFromArticle: boolean, includeMetadata: boolean): MetadataBlock | undefined;
|
|
23
|
+
declare function findContentRoot(document: Document): string | undefined;
|
|
24
|
+
declare function findPrimaryHeading(document: Document): string | undefined;
|
|
25
|
+
declare function isGithubRepositoryRootUrl(url: string): boolean;
|
|
26
|
+
export declare const TransformHeuristics: {
|
|
27
|
+
readonly findContentRoot: typeof findContentRoot;
|
|
28
|
+
readonly findPrimaryHeading: typeof findPrimaryHeading;
|
|
29
|
+
readonly isGithubRepositoryRootUrl: typeof isGithubRepositoryRootUrl;
|
|
30
|
+
};
|
|
23
31
|
export declare function transformHtmlToMarkdownInProcess(html: string, url: string, options: TransformOptions): MarkdownTransformResult;
|
|
24
32
|
interface TransformPoolStats {
|
|
25
33
|
queueDepth: number;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"transform.d.ts","sourceRoot":"","sources":["../../src/transform/transform.ts"],"names":[],"mappings":"AAuCA,OAAO,KAAK,EACV,gBAAgB,EAChB,iBAAiB,EACjB,gBAAgB,EAChB,uBAAuB,EACvB,aAAa,EACb,gBAAgB,EAChB,qBAAqB,EAEtB,MAAM,YAAY,CAAC;AAqCpB,UAAU,WAAW;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;CACnB;
|
|
1
|
+
{"version":3,"file":"transform.d.ts","sourceRoot":"","sources":["../../src/transform/transform.ts"],"names":[],"mappings":"AAuCA,OAAO,KAAK,EACV,gBAAgB,EAChB,iBAAiB,EACjB,gBAAgB,EAChB,uBAAuB,EACvB,aAAa,EACb,gBAAgB,EAChB,qBAAqB,EAEtB,MAAM,YAAY,CAAC;AAqCpB,UAAU,WAAW;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;CACnB;AAuJD,wBAAgB,mBAAmB,CACjC,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,EACb,MAAM,CAAC,EAAE,WAAW,GACnB,qBAAqB,GAAG,IAAI,CAE9B;AAED,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,qBAAqB,GAAG,IAAI,EACrC,OAAO,CAAC,EAAE;IAAE,SAAS,CAAC,EAAE,OAAO,CAAA;CAAE,GAChC,MAAM,CAER;AAuVD,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,GAAE;IAAE,cAAc,CAAC,EAAE,OAAO,CAAC;IAAC,MAAM,CAAC,EAAE,WAAW,CAAA;CAExD,GACA,gBAAgB,CAGlB;AA+OD,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,QAAQ,CAAC,EAAE,aAAa,EACxB,OAAO,CAAC,EAAE;IACR,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B,GACA,MAAM,CAsBR;AAwJD,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,gBAAgB,GAAG,IAAI,EAChC,sBAAsB,EAAE,MAAM,GAAG,QAAQ,GACxC,OAAO,CAQT;AAiED,wBAAgB,gCAAgC,CAC9C,OAAO,EAAE,gBAAgB,GAAG,IAAI,GAC/B,OAAO,IAAI,gBAAgB,CAE7B;AAED,wBAAgB,0BAA0B,CACxC,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GAAG,IAAI,EAChC,aAAa,EAAE,iBAAiB,EAChC,wBAAwB,EAAE,OAAO,EACjC,eAAe,EAAE,OAAO,GACvB,aAAa,GAAG,SAAS,CAuB3B;AAsCD,iBAAS,eAAe,CAAC,QAAQ,EAAE,QAAQ,GAAG,MAAM,GAAG,SAAS,CAc/D;AAED,iBAAS,kBAAkB,CAAC,QAAQ,EAAE,QAAQ,GAAG,MAAM,GAAG,SAAS,CAYlE;AAED,iBAAS,yBAAyB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAcvD;AAED,eAAO,MAAM,mBAAmB;;;;CAItB,CAAC;AAiQX,wBAAgB,gCAAgC,CAC9C,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GACxB,uBAAuB,CAmBzB;AA+CD,UAAU,kBAAkB;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,wBAAgB,qBAAqB,IAAI,kBAAkB,GAAG,IAAI,CAEjE;AAED,wBAAsB,2BAA2B,IAAI,OAAO,CAAC,IAAI,CAAC,CAEjE;AAED,KAAK,yBAAyB,GAAG,gBAAgB,GAAG;IAAE,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAsH1E,wBAAsB,uBAAuB,CAC3C,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,uBAAuB,CAAC,CAElC;AAED,wBAAsB,yBAAyB,CAC7C,UAAU,EAAE,UAAU,EACtB,GAAG,EAAE,MAAM,EACX,OAAO,EAAE,yBAAyB,GACjC,OAAO,CAAC,uBAAuB,CAAC,CAElC"}
|