@j0hanz/fetch-url-mcp 1.8.1 → 1.8.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (9) hide show
  1. package/dist/lib/fetch-pipeline.d.ts +3 -0
  2. package/dist/lib/fetch-pipeline.d.ts.map +1 -1
  3. package/dist/lib/fetch-pipeline.js +8 -0
  4. package/dist/lib/mcp-tools.d.ts +1 -1
  5. package/dist/lib/mcp-tools.d.ts.map +1 -1
  6. package/dist/lib/progress.js +1 -1
  7. package/dist/tools/fetch-url.d.ts.map +1 -1
  8. package/dist/tools/fetch-url.js +29 -11
  9. package/package.json +1 -1
package/dist/lib/fetch-pipeline.d.ts CHANGED
@@ -17,6 +17,7 @@ interface FetchPipelineOptions<T> {
17
17
  signal?: AbortSignal;
18
18
  cacheVary?: Record<string, unknown> | string;
19
19
  forceRefresh?: boolean;
20
+ onStage?: (stage: SharedFetchStage) => void;
20
21
  transform: (input: {
21
22
  buffer: Uint8Array;
22
23
  encoding: string;
@@ -34,6 +35,7 @@ export interface PipelineResult<T> {
34
35
  fetchedAt: string;
35
36
  cacheKey?: string | null;
36
37
  }
38
+ export type SharedFetchStage = 'resolve_url' | 'check_cache' | 'cache_hit' | 'fetch_remote' | 'response_ready' | 'transform_start' | 'finalize_output';
37
39
  export declare function executeFetchPipeline<T>(options: FetchPipelineOptions<T>): Promise<PipelineResult<T>>;
38
40
  export type MarkdownPipelineResult = MarkdownTransformResult & {
39
41
  readonly content: string;
@@ -51,6 +53,7 @@ interface SharedFetchOptions {
51
53
  readonly cacheVary?: Record<string, unknown> | string;
52
54
  readonly forceRefresh?: boolean;
53
55
  readonly maxInlineChars?: number;
56
+ readonly onStage?: (stage: SharedFetchStage) => void;
54
57
  readonly transform: (input: {
55
58
  buffer: Uint8Array;
56
59
  encoding: string;
package/dist/lib/fetch-pipeline.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"fetch-pipeline.d.ts","sourceRoot":"","sources":["../../src/lib/fetch-pipeline.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,KAAK,uBAAuB,EAAE,MAAM,uBAAuB,CAAC;AAqBrE,KAAK,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAY1C,wBAAgB,gBAAgB,CAC9B,GAAG,EAAE,OAAO,EACZ,IAAI,EAAE,SAAS,MAAM,EAAE,GACtB,UAAU,GAAG,SAAS,CAOxB;AACD,wBAAgB,UAAU,CACxB,MAAM,CAAC,EAAE,WAAW,GACnB;IAAE,MAAM,EAAE,WAAW,CAAA;CAAE,GAAG,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAEjD;AAMD,eAAO,MAAM,iBAAiB,mBAAmB,CAAC;AAClD,MAAM,WAAW,mBAAmB;IAClC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAuED,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,MAAM,GACb,MAAM,CAkBR;AAoDD,UAAU,oBAAoB,CAAC,CAAC;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,CAAC;IAC7C,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,SAAS,EAAE,CACT,KAAK,EAAE;QAAE,MAAM,EAAE,UAAU,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,OAAO,CAAA;KAAE,EACpE,GAAG,EAAE,MAAM,KACR,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IACpB,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,MAAM,CAAC;IAClC,WAAW,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,CAAC,GAAG,SAAS,CAAC;CACjD;AACD,MAAM,WAAW,cAAc,CAAC,CAAC;IAC/B,IAAI,EAAE,CAAC,CAAC;IACR,SAAS,EAAE,OAAO,CAAC;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B;AAmMD,wBAAsB,oBAAoB,CAAC,CAAC,EAC1C,OAAO,EAAE,oBAAoB,CAAC,CAAC,CAAC,GAC/B,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CA4D5B;AAMD,MAAM,MAAM,sBAAsB,GAAG,uBAAuB,GAAG;IAC7D,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B,CAAC;AACF,wBAAgB,yBAAyB,CACvC,MAAM,EAAE,MAAM,GACb,sBAAsB,GAAG,SAAS,CAqBpC;AACD,eAAO,MAAM,iBAAiB,GAC5B,OAAO;IAAE,MAAM,EAAE,UAAU,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,OAAO,CAAA;CAAE,EACpE,KAAK,MAAM,EACX,SAAS,WAAW,EACpB,mBAAmB,OAAO,KACzB,OAAO,CAAC,sBAAsB,CAUhC,CAAC;AACF,wBAAgB,uBAAuB,CACrC,MAAM,EAAE,sBAAsB,GAC7B,MAAM,CAaR;AAMD,UAAU,kBAAkB;IAC1B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,CAAC;IAC9B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,CAAC;IACtD,QAAQ,CAAC,YAAY,CAAC,EAAE,OAAO,CAAC;IAChC,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,SAAS,EAAE,CAClB,KAAK,EAAE;QAAE,MAAM,EAAE,UAAU,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,OAAO,CAAA;KAAE,EACpE,aAAa,EAAE,MAAM,KAClB,sBAAsB,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAAC;IAC9D,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,sBAAsB,KAAK,MAAM,CAAC;IAChE,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,sBAAsB,GAAG,SAAS,CAAC;CAC/E;AACD,UAAU,eAAe;IACvB,QAAQ,CAAC,oBAAoB,CAAC,EAAE,OAAO,oBAAoB,CAAC;CAC7D;AAeD,wBAAsB,kBAAkB,CACtC,OAAO,EAAE,kBAAkB,EAC3B,IAAI,GAAE,eAAoB,GACzB,OAAO,CAAC;IACT,QAAQ,EAAE,cAAc,CAAC,sBAAsB,CAAC,CAAC;IACjD,YAAY,EAAE,mBAAmB,CAAC;CACnC,CAAC,CAWD"}
1
+ {"version":3,"file":"fetch-pipeline.d.ts","sourceRoot":"","sources":["../../src/lib/fetch-pipeline.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,KAAK,uBAAuB,EAAE,MAAM,uBAAuB,CAAC;AAqBrE,KAAK,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAY1C,wBAAgB,gBAAgB,CAC9B,GAAG,EAAE,OAAO,EACZ,IAAI,EAAE,SAAS,MAAM,EAAE,GACtB,UAAU,GAAG,SAAS,CAOxB;AACD,wBAAgB,UAAU,CACxB,MAAM,CAAC,EAAE,WAAW,GACnB;IAAE,MAAM,EAAE,WAAW,CAAA;CAAE,GAAG,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAEjD;AAMD,eAAO,MAAM,iBAAiB,mBAAmB,CAAC;AAClD,MAAM,WAAW,mBAAmB;IAClC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAuED,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,MAAM,GACb,MAAM,CAkBR;AAoDD,UAAU,oBAAoB,CAAC,CAAC;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,CAAC;IAC7C,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,gBAAgB,KAAK,IAAI,CAAC;IAC5C,SAAS,EAAE,CACT,KAAK,EAAE;QAAE,MAAM,EAAE,UAAU,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,OAAO,CAAA;KAAE,EACpE,GAAG,EAAE,MAAM,KACR,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IACpB,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,MAAM,CAAC;IAClC,WAAW,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,CAAC,GAAG,SAAS,CAAC;CACjD;AACD,MAAM,WAAW,cAAc,CAAC,CAAC;IAC/B,IAAI,EAAE,CAAC,CAAC;IACR,SAAS,EAAE,OAAO,CAAC;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B;AACD,MAAM,MAAM,gBAAgB,GACxB,aAAa,GACb,aAAa,GACb,WAAW,GACX,cAAc,GACd,gBAAgB,GAChB,iBAAiB,GACjB,iBAAiB,CAAC;AAmMtB,wBAAsB,oBAAoB,CAAC,CAAC,EAC1C,OAAO,EAAE,oBAAoB,CAAC,CAAC,CAAC,GAC/B,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAkE5B;AAMD,MAAM,MAAM,sBAAsB,GAAG,uBAAuB,GAAG;IAC7D,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B,CAAC;AACF,wBAAgB,yBAAyB,CACvC,MAAM,EAAE,MAAM,GACb,sBAAsB,GAAG,SAAS,CAqBpC;AACD,eAAO,MAAM,iBAAiB,GAC5B,OAAO;IAAE,MAAM,EAAE,UAAU,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,OAAO,CAAA;CAAE,EACpE,KAAK,MAAM,EACX,SAAS,WAAW,EACpB,mBAAmB,OAAO,KACzB,OAAO,CAAC,sBAAsB,CAUhC,CAAC;AACF,wBAAgB,uBAAuB,CACrC,MAAM,EAAE,sBAAsB,GAC7B,MAAM,CAaR;AAMD,UAAU,kBAAkB;IAC1B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,CAAC;IAC9B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,CAAC;IACtD,QAAQ,CAAC,YAAY,CAAC,EAAE,OAAO,CAAC;IAChC,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,gBAAgB,KAAK,IAAI,CAAC;IACrD,QAAQ,CAAC,SAAS,EAAE,CAClB,KAAK,EAAE;QAAE,MAAM,EAAE,UAAU,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,OAAO,CAAA;KAAE,EACpE,aAAa,EAAE,MAAM,KAClB,sBAAsB,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAAC;IAC9D,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,sBAAsB,KAAK,MAAM,CAAC;IAChE,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,sBAAsB,GAAG,SAAS,CAAC;CAC/E;AACD,UAAU,eAAe;IACvB,QAAQ,CAAC,oBAAoB,CAAC,EAAE,OAAO,oBAAoB,CAAC;CAC7D;AAgBD,wBAAsB,kBAAkB,CACtC,OAAO,EAAE,kBAAkB,EAC3B,IAAI,GAAE,eAAoB,GACzB,OAAO,CAAC;IACT,QAAQ,EAAE,cAAc,CAAC,sBAAsB,CAAC,CAAC;IACjD,YAAY,EAAE,mBAAmB,CAAC;CACnC,CAAC,CAYD"}
package/dist/lib/fetch-pipeline.js CHANGED
@@ -271,10 +271,12 @@ function persistCacheTargets(params) {
271
271
  }
272
272
  }
273
273
  export async function executeFetchPipeline(options) {
274
+ options.onStage?.('resolve_url');
274
275
  const resolvedUrl = resolveNormalizedUrl(options.url);
275
276
  logRawUrlTransformation(resolvedUrl);
276
277
  const cacheKey = createCacheKey(options.cacheNamespace, resolvedUrl.normalizedUrl, options.cacheVary);
277
278
  if (!options.forceRefresh) {
279
+ options.onStage?.('check_cache');
278
280
  const cachedResult = attemptCacheRetrieval({
279
281
  cacheKey,
280
282
  deserialize: options.deserialize,
@@ -282,13 +284,17 @@ export async function executeFetchPipeline(options) {
282
284
  normalizedUrl: resolvedUrl.normalizedUrl,
283
285
  });
284
286
  if (cachedResult) {
287
+ options.onStage?.('cache_hit');
285
288
  return { ...cachedResult, originalUrl: resolvedUrl.originalUrl };
286
289
  }
287
290
  }
291
+ options.onStage?.('fetch_remote');
288
292
  logDebug('Fetching URL', { url: resolvedUrl.normalizedUrl });
289
293
  const { buffer, encoding, truncated, finalUrl } = await fetchNormalizedUrlBuffer(resolvedUrl.normalizedUrl, withSignal(options.signal));
294
+ options.onStage?.('response_ready');
290
295
  const resolvedFinalUrl = finalUrl || resolvedUrl.normalizedUrl;
291
296
  const transformUrl = resolvedFinalUrl;
297
+ options.onStage?.('transform_start');
292
298
  const data = await options.transform({ buffer, encoding, ...(truncated ? { truncated: true } : {}) }, transformUrl);
293
299
  if (isEnabled()) {
294
300
  persistCacheTargets({
@@ -364,6 +370,7 @@ function buildSharedFetchPipelineOptions(options) {
364
370
  ...withSignal(options.signal),
365
371
  ...(options.cacheVary ? { cacheVary: options.cacheVary } : {}),
366
372
  ...(options.forceRefresh ? { forceRefresh: true } : {}),
373
+ ...(options.onStage ? { onStage: options.onStage } : {}),
367
374
  transform: options.transform,
368
375
  ...(options.serialize ? { serialize: options.serialize } : {}),
369
376
  ...(options.deserialize ? { deserialize: options.deserialize } : {}),
@@ -372,6 +379,7 @@ function buildSharedFetchPipelineOptions(options) {
372
379
  export async function performSharedFetch(options, deps = {}) {
373
380
  const executePipeline = deps.executeFetchPipeline ?? executeFetchPipeline;
374
381
  const pipeline = await executePipeline(buildSharedFetchPipelineOptions(options));
382
+ options.onStage?.('finalize_output');
375
383
  const inlineResult = applyInlineContentLimit(pipeline.data.content, options.maxInlineChars);
376
384
  return { pipeline, inlineResult };
377
385
  }
package/dist/lib/mcp-tools.d.ts CHANGED
@@ -61,6 +61,6 @@ export declare function createToolErrorResponse(message: string, url: string, ex
61
61
  }): ToolErrorResponse;
62
62
  export declare function handleToolError(error: unknown, url: string, fallbackMessage?: string): ToolErrorResponse;
63
63
  export { registerServerLifecycleCleanup, registerTaskHandlers, cancelTasksForOwner, abortAllTaskExecutions, } from './task-handlers.js';
64
- export { readNestedRecord, withSignal, TRUNCATION_MARKER, type InlineContentResult, appendTruncationMarker, type PipelineResult, executeFetchPipeline, type MarkdownPipelineResult, parseCachedMarkdownResult, markdownTransform, serializeMarkdownResult, performSharedFetch, } from './fetch-pipeline.js';
64
+ export { readNestedRecord, withSignal, TRUNCATION_MARKER, type InlineContentResult, appendTruncationMarker, type PipelineResult, type SharedFetchStage, executeFetchPipeline, type MarkdownPipelineResult, parseCachedMarkdownResult, markdownTransform, serializeMarkdownResult, performSharedFetch, } from './fetch-pipeline.js';
65
65
  export { type ProgressNotificationParams, type ProgressNotification, type ToolHandlerExtra, type ProgressReporter, createProgressReporter, } from './progress.js';
66
66
  //# sourceMappingURL=mcp-tools.d.ts.map
package/dist/lib/mcp-tools.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"mcp-tools.d.ts","sourceRoot":"","sources":["../../src/lib/mcp-tools.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,cAAc,EAAE,MAAM,oCAAoC,CAAC;AACzE,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAQxB,MAAM,MAAM,SAAS,GAAG,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;AAC/C,UAAU,gBAAgB;IACxB,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChC,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAGD,QAAA,MAAM,oBAAoB;;;;;kBAKxB,CAAC;AAeH,QAAA,MAAM,qBAAqB;;;;;;;;;;;;oBAGzB,CAAC;AACH,QAAA,MAAM,oBAAoB;;;;;;;;;;;;;;;;;sBAGxB,CAAC;AACH,KAAK,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAC3D,KAAK,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACjE,KAAK,kBAAkB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAC/D,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,OAAO,GAAG,OAAO,CAE5D;AACD,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,OAAO,GAAG,IAAI,IAAI,cAAc,CAEtE;AACD,wBAAgB,qBAAqB,CACnC,IAAI,EAAE,OAAO,GACZ,IAAI,IAAI,mBAAmB,CAE7B;AACD,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,OAAO,GAAG,IAAI,IAAI,kBAAkB,CAE1E;AAaD,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,OAAO,CAG7E;AAeD,wBAAgB,yBAAyB,CACvC,MAAM,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAChC,OAAO,CAUT;AAMD,KAAK,iBAAiB,GAAG,cAAc,GAAG;IACxC,OAAO,EAAE,IAAI,CAAC;CACf,CAAC;AAsCF,wBAAgB,uBAAuB,CACrC,OAAO,EAAE,MAAM,EACf,GAAG,EAAE,MAAM,EACX,KAAK,CAAC,EAAE;IACN,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC,GACA,iBAAiB,CAenB;AAiCD,wBAAgB,eAAe,CAC7B,KAAK,EAAE,OAAO,EACd,GAAG,EAAE,MAAM,EACX,eAAe,SAAqB,GACnC,iBAAiB,CAYnB;AASD,OAAO,EACL,8BAA8B,EAC9B,oBAAoB,EACpB,mBAAmB,EACnB,sBAAsB,GACvB,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EACL,gBAAgB,EAChB,UAAU,EACV,iBAAiB,EACjB,KAAK,mBAAmB,EACxB,sBAAsB,EACtB,KAAK,cAAc,EACnB,oBAAoB,EACpB,KAAK,sBAAsB,EAC3B,yBAAyB,EACzB,iBAAiB,EACjB,uBAAuB,EACvB,kBAAkB,GACnB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EACL,KAAK,0BAA0B,EAC/B,KAAK,oBAAoB,EACzB,KAAK,gBAAgB,EACrB,KAAK,gBAAgB,EACrB,sBAAsB,GACvB,MAAM,eAAe,CAAC"}
1
+ {"version":3,"file":"mcp-tools.d.ts","sourceRoot":"","sources":["../../src/lib/mcp-tools.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,cAAc,EAAE,MAAM,oCAAoC,CAAC;AACzE,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAQxB,MAAM,MAAM,SAAS,GAAG,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC;AAC/C,UAAU,gBAAgB;IACxB,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChC,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAGD,QAAA,MAAM,oBAAoB;;;;;kBAKxB,CAAC;AAeH,QAAA,MAAM,qBAAqB;;;;;;;;;;;;oBAGzB,CAAC;AACH,QAAA,MAAM,oBAAoB;;;;;;;;;;;;;;;;;sBAGxB,CAAC;AACH,KAAK,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAC3D,KAAK,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACjE,KAAK,kBAAkB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAC/D,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,OAAO,GAAG,OAAO,CAE5D;AACD,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,OAAO,GAAG,IAAI,IAAI,cAAc,CAEtE;AACD,wBAAgB,qBAAqB,CACnC,IAAI,EAAE,OAAO,GACZ,IAAI,IAAI,mBAAmB,CAE7B;AACD,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,OAAO,GAAG,IAAI,IAAI,kBAAkB,CAE1E;AAaD,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,OAAO,CAG7E;AAeD,wBAAgB,yBAAyB,CACvC,MAAM,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAChC,OAAO,CAUT;AAMD,KAAK,iBAAiB,GAAG,cAAc,GAAG;IACxC,OAAO,EAAE,IAAI,CAAC;CACf,CAAC;AAsCF,wBAAgB,uBAAuB,CACrC,OAAO,EAAE,MAAM,EACf,GAAG,EAAE,MAAM,EACX,KAAK,CAAC,EAAE;IACN,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC,GACA,iBAAiB,CAenB;AAiCD,wBAAgB,eAAe,CAC7B,KAAK,EAAE,OAAO,EACd,GAAG,EAAE,MAAM,EACX,eAAe,SAAqB,GACnC,iBAAiB,CAYnB;AASD,OAAO,EACL,8BAA8B,EAC9B,oBAAoB,EACpB,mBAAmB,EACnB,sBAAsB,GACvB,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EACL,gBAAgB,EAChB,UAAU,EACV,iBAAiB,EACjB,KAAK,mBAAmB,EACxB,sBAAsB,EACtB,KAAK,cAAc,EACnB,KAAK,gBAAgB,EACrB,oBAAoB,EACpB,KAAK,sBAAsB,EAC3B,yBAAyB,EACzB,iBAAiB,EACjB,uBAAuB,EACvB,kBAAkB,GACnB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EACL,KAAK,0BAA0B,EAC/B,KAAK,oBAAoB,EACzB,KAAK,gBAAgB,EACrB,KAAK,gBAAgB,EACrB,sBAAsB,GACvB,MAAM,eAAe,CAAC"}
package/dist/lib/progress.js CHANGED
@@ -1,6 +1,6 @@
1
1
  import { logWarn } from './core.js';
2
2
  import { getErrorMessage, isObject } from './utils.js';
3
- const FETCH_PROGRESS_TOTAL = 4;
3
+ const FETCH_PROGRESS_TOTAL = 8;
4
4
  const PROGRESS_NOTIFICATION_TIMEOUT_MS = 5000;
5
5
  function resolveRelatedTaskMeta(meta) {
6
6
  if (!meta)
package/dist/tools/fetch-url.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"fetch-url.d.ts","sourceRoot":"","sources":["../../src/tools/fetch-url.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACV,SAAS,EAEV,MAAM,yCAAyC,CAAC;AACjD,OAAO,KAAK,EACV,YAAY,EAEb,MAAM,oCAAoC,CAAC;AAE5C,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAwBxB,OAAO,EAGL,KAAK,gBAAgB,EACtB,MAAM,qBAAqB,CAAC;AAI7B,OAAO,EACL,mBAAmB,EAIpB,MAAM,eAAe,CAAC;AAMvB,KAAK,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAEzD,KAAK,qBAAqB,GAAG,YAAY,CAAC;AAE1C,UAAU,gBAAgB;IACxB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;IACvB,OAAO,EAAE,qBAAqB,EAAE,CAAC;IACjC,iBAAiB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,CAAC;IACxD,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,eAAO,MAAM,mBAAmB,cAAc,CAAC;AA4S/C,wBAAsB,mBAAmB,CACvC,KAAK,EAAE,aAAa,EACpB,KAAK,CAAC,EAAE,gBAAgB,GACvB,OAAO,CAAC,gBAAgB,CAAC,CAK3B;AAgDD;;;;;;GAMG;AACH,wBAAgB,2BAA2B,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,EAC5E,OAAO,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,GAC7D,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAmBvD;AAwBD,wBAAgB,aAAa,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI,CAwCrD"}
1
+ {"version":3,"file":"fetch-url.d.ts","sourceRoot":"","sources":["../../src/tools/fetch-url.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EACV,SAAS,EAEV,MAAM,yCAAyC,CAAC;AACjD,OAAO,KAAK,EACV,YAAY,EAEb,MAAM,oCAAoC,CAAC;AAE5C,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAyBxB,OAAO,EAGL,KAAK,gBAAgB,EACtB,MAAM,qBAAqB,CAAC;AAI7B,OAAO,EACL,mBAAmB,EAIpB,MAAM,eAAe,CAAC;AAMvB,KAAK,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAEzD,KAAK,qBAAqB,GAAG,YAAY,CAAC;AAE1C,UAAU,gBAAgB;IACxB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;IACvB,OAAO,EAAE,qBAAqB,EAAE,CAAC;IACjC,iBAAiB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,CAAC;IACxD,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,eAAO,MAAM,mBAAmB,cAAc,CAAC;AAqU/C,wBAAsB,mBAAmB,CACvC,KAAK,EAAE,aAAa,EACpB,KAAK,CAAC,EAAE,gBAAgB,GACvB,OAAO,CAAC,gBAAgB,CAAC,CAK3B;AAgDD;;;;;;GAMG;AACH,wBAAgB,2BAA2B,CAAC,OAAO,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,EAC5E,OAAO,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,GAC7D,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAmBvD;AAwBD,wBAAgB,aAAa,CAAC,MAAM,EAAE,SAAS,GAAG,IAAI,CAwCrD"}
package/dist/tools/fetch-url.js CHANGED
@@ -157,23 +157,44 @@ function getUrlContext(urlStr) {
157
157
  return 'unknown';
158
158
  }
159
159
  }
160
- function buildFetchOptions(url, signal, progress, skipNoiseRemoval, forceRefresh, maxInlineChars) {
160
+ function mapFetchStageToProgress(stage, context) {
161
+ switch (stage) {
162
+ case 'resolve_url':
163
+ return { step: 2, message: 'Resolving URL' };
164
+ case 'check_cache':
165
+ return { step: 3, message: 'Checking cache' };
166
+ case 'cache_hit':
167
+ return { step: 4, message: 'Loaded from cache' };
168
+ case 'fetch_remote':
169
+ return { step: 4, message: `Fetching ${context}` };
170
+ case 'response_ready':
171
+ return { step: 5, message: 'Received response' };
172
+ case 'transform_start':
173
+ return { step: 6, message: 'Parsing HTML → Markdown' };
174
+ case 'finalize_output':
175
+ return { step: 7, message: 'Finalizing output' };
176
+ }
177
+ }
178
+ function buildFetchOptions(url, context, signal, progress, skipNoiseRemoval, forceRefresh, maxInlineChars) {
161
179
  return {
162
180
  url,
163
181
  ...withSignal(signal),
164
182
  ...(skipNoiseRemoval ? { cacheVary: { skipNoiseRemoval: true } } : {}),
165
183
  ...(forceRefresh ? { forceRefresh: true } : {}),
166
184
  ...(maxInlineChars !== undefined ? { maxInlineChars } : {}),
185
+ onStage: (stage) => {
186
+ const update = mapFetchStageToProgress(stage, context);
187
+ reportProgress(progress, update.step, update.message);
188
+ },
167
189
  transform: async ({ buffer, encoding, truncated }, normalizedUrl) => {
168
- reportProgress(progress, 2, 'Parsing HTML → Markdown');
169
190
  return markdownTransform({ buffer, encoding, ...(truncated ? { truncated } : {}) }, normalizedUrl, signal, skipNoiseRemoval);
170
191
  },
171
192
  serialize: serializeMarkdownResult,
172
193
  deserialize: parseCachedMarkdownResult,
173
194
  };
174
195
  }
175
- async function fetchPipeline(url, signal, progress, skipNoiseRemoval, forceRefresh, maxInlineChars) {
176
- return performSharedFetch(buildFetchOptions(url, signal, progress, skipNoiseRemoval, forceRefresh, maxInlineChars));
196
+ async function fetchPipeline(url, context, signal, progress, skipNoiseRemoval, forceRefresh, maxInlineChars) {
197
+ return performSharedFetch(buildFetchOptions(url, context, signal, progress, skipNoiseRemoval, forceRefresh, maxInlineChars));
177
198
  }
178
199
  function formatContentSize(chars) {
179
200
  if (chars < 1000)
@@ -194,18 +215,15 @@ async function executeFetch(input, extra) {
194
215
  const context = getUrlContext(url);
195
216
  logDebug('Fetching URL', { url });
196
217
  try {
197
- reportProgress(progress, 1, `Fetching ${context}`);
198
- const { pipeline, inlineResult } = await fetchPipeline(url, signal, progress, input.skipNoiseRemoval, input.forceRefresh, input.maxInlineChars);
199
- if (pipeline.fromCache) {
200
- reportProgress(progress, 3, 'Loaded from cache');
201
- }
218
+ reportProgress(progress, 1, 'Preparing request');
219
+ const { pipeline, inlineResult } = await fetchPipeline(url, context, signal, progress, input.skipNoiseRemoval, input.forceRefresh, input.maxInlineChars);
202
220
  const size = formatContentSize(inlineResult.contentSize);
203
- reportProgress(progress, 4, `Done — ${size}`);
221
+ reportProgress(progress, 8, `Done — ${size}`);
204
222
  return buildResponse(pipeline, inlineResult, url);
205
223
  }
206
224
  catch (error) {
207
225
  const isAbort = isAbortError(error);
208
- reportProgress(progress, 4, isAbort ? 'Cancelled' : 'Failed');
226
+ reportProgress(progress, 8, isAbort ? 'Cancelled' : 'Failed');
209
227
  throw error;
210
228
  }
211
229
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@j0hanz/fetch-url-mcp",
3
- "version": "1.8.1",
3
+ "version": "1.8.2",
4
4
  "mcpName": "io.github.j0hanz/fetch-url-mcp",
5
5
  "description": "A web content fetcher MCP server that converts HTML to clean, AI and human readable markdown.",
6
6
  "type": "module",