@j0hanz/fetch-url-mcp 1.9.1 → 1.9.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. package/dist/http/auth.d.ts +0 -1
  2. package/dist/http/auth.d.ts.map +1 -1
  3. package/dist/http/auth.js +1 -13
  4. package/dist/http/native.d.ts.map +1 -1
  5. package/dist/http/native.js +2 -5
  6. package/dist/lib/content.d.ts.map +1 -1
  7. package/dist/lib/content.js +301 -350
  8. package/dist/lib/core.d.ts +78 -71
  9. package/dist/lib/core.d.ts.map +1 -1
  10. package/dist/lib/core.js +308 -372
  11. package/dist/lib/fetch-pipeline.d.ts +2 -6
  12. package/dist/lib/fetch-pipeline.d.ts.map +1 -1
  13. package/dist/lib/fetch-pipeline.js +51 -137
  14. package/dist/lib/http.d.ts.map +1 -1
  15. package/dist/lib/http.js +188 -130
  16. package/dist/lib/mcp-tools.d.ts +3 -5
  17. package/dist/lib/mcp-tools.d.ts.map +1 -1
  18. package/dist/lib/mcp-tools.js +22 -58
  19. package/dist/lib/task-handlers.js +4 -4
  20. package/dist/lib/utils.d.ts +6 -0
  21. package/dist/lib/utils.d.ts.map +1 -1
  22. package/dist/lib/utils.js +23 -0
  23. package/dist/resources/index.js +1 -1
  24. package/dist/schemas.d.ts +0 -1
  25. package/dist/schemas.d.ts.map +1 -1
  26. package/dist/schemas.js +4 -6
  27. package/dist/server.js +1 -1
  28. package/dist/tasks/owner.d.ts +1 -1
  29. package/dist/tasks/owner.d.ts.map +1 -1
  30. package/dist/tasks/tool-registry.d.ts +1 -1
  31. package/dist/tasks/tool-registry.d.ts.map +1 -1
  32. package/dist/tools/fetch-url.d.ts +2 -3
  33. package/dist/tools/fetch-url.d.ts.map +1 -1
  34. package/dist/tools/fetch-url.js +89 -152
  35. package/dist/transform/transform.d.ts +8 -0
  36. package/dist/transform/transform.d.ts.map +1 -1
  37. package/dist/transform/transform.js +109 -108
  38. package/dist/transform/worker-pool.d.ts +3 -6
  39. package/dist/transform/worker-pool.d.ts.map +1 -1
  40. package/dist/transform/worker-pool.js +148 -118
  41. package/package.json +2 -1
@@ -1,9 +1,6 @@
1
1
  import { type MarkdownTransformResult } from '../transform/types.js';
2
- type JsonRecord = Record<string, unknown>;
3
- export declare function readNestedRecord(obj: unknown, keys: readonly string[]): JsonRecord | undefined;
4
- export declare function withSignal(signal?: AbortSignal): {
5
- signal: AbortSignal;
6
- } | Record<string, never>;
2
+ import { readNestedRecord, withSignal } from './utils.js';
3
+ export { readNestedRecord, withSignal };
7
4
  export declare const TRUNCATION_MARKER = "...[truncated]";
8
5
  export interface InlineContentResult {
9
6
  content?: string;
@@ -68,5 +65,4 @@ export declare function performSharedFetch(options: SharedFetchOptions, deps?: S
68
65
  pipeline: PipelineResult<MarkdownPipelineResult>;
69
66
  inlineResult: InlineContentResult;
70
67
  }>;
71
- export {};
72
68
  //# sourceMappingURL=fetch-pipeline.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"fetch-pipeline.d.ts","sourceRoot":"","sources":["../../src/lib/fetch-pipeline.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,KAAK,uBAAuB,EAAE,MAAM,uBAAuB,CAAC;AAqBrE,KAAK,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAY1C,wBAAgB,gBAAgB,CAC9B,GAAG,EAAE,OAAO,EACZ,IAAI,EAAE,SAAS,MAAM,EAAE,GACtB,UAAU,GAAG,SAAS,CAOxB;AACD,wBAAgB,UAAU,CACxB,MAAM,CAAC,EAAE,WAAW,GACnB;IAAE,MAAM,EAAE,WAAW,CAAA;CAAE,GAAG,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAEjD;AAMD,eAAO,MAAM,iBAAiB,mBAAmB,CAAC;AAClD,MAAM,WAAW,mBAAmB;IAClC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAuED,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,MAAM,GACb,MAAM,CAkBR;AAsCD,UAAU,oBAAoB,CAAC,CAAC;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,CAAC;IAC7C,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,gBAAgB,KAAK,IAAI,CAAC;IAC5C,SAAS,EAAE,CACT,KAAK,EAAE;QAAE,MAAM,EAAE,UAAU,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,OAAO,CAAA;KAAE,EACpE,GAAG,EAAE,MAAM,KACR,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IACpB,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,MAAM,CAAC;IAClC,WAAW,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,CAAC,GAAG,SAAS,CAAC;CACjD;AACD,MAAM,WAAW,cAAc,CAAC,CAAC;IAC/B,IAAI,EAAE,CAAC,CAAC;IACR,SAAS,EAAE,OAAO,CAAC;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B;AACD,MAAM,MAAM,gBAAgB,GACxB,aAAa,GACb,aAAa,GACb,WAAW,GACX,eAAe,GACf,cAAc,GACd,gBAAgB,GAChB,iBAAiB,GACjB,gBAAgB,GAChB,iBAAiB,CAAC;AAmMtB,wBAAsB,oBAAoB,CAAC,CAAC,EAC1C,OAAO,EAAE,oBAAoB,CAAC,CAAC,CAAC,GAC/B,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAmE5B;AAMD,MAAM,MAAM,sBAAsB,GAAG,uBAAuB,GAAG;IAC7D,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B,CAAC;AACF,wBAAgB,yBAAyB,CACvC,MAAM,EAAE,MAAM,GACb,sBAAsB,GAAG,SAAS,CAqBpC;AACD,eAAO,MAAM,iBAAiB,GAC5B,OAAO;IAAE,MAAM,EAAE,UAAU,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,OAAO,CAAA;CAAE,EACpE,KAAK,MAAM,EACX,SAAS,WAAW,KACnB,OAAO,CAAC,sBAAsB,CAShC,CAAC;AACF,wBAAgB,uBAAuB,CACrC,MAAM,EAAE,sBAAsB,GAC7B,MAAM,CAaR;AAMD,UAAU,kBAAkB;IAC1B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,CAAC;IAC9B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,CAAC;IACtD,QAAQ,CAAC,YAAY,CAAC,EAAE,OAAO,CAAC;IAChC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,gBAAgB,KAAK,IAAI,CAAC;IACrD,QAAQ,CAAC,SAAS,EAAE,CAClB,KAAK,EAAE;QAAE,MAAM,EAAE,UAAU,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,OAAO,CAAA;KAAE,EACpE,aAAa,EAAE,MAAM,KAClB,sBAAsB,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAAC;IAC9D,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,sBAAsB,KAAK,MAAM,CAAC;IAChE,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,sBAAsB,GAAG,SAAS,CAAC;CAC/E;AACD,UAAU,eAAe;IACvB,QAAQ,CAAC,oBAAoB,CAAC,EAAE,OAAO,oBAAoB,CAAC;CAC7D;AAgBD,wBAAsB,kBAAkB,CACtC,OAAO,EAAE,kBAAkB,EAC3B,IAAI,GAAE,eAAoB,GACzB,OAAO,CAAC;IACT,QAAQ,EAAE,cAAc,CAAC,sBAAsB,CAAC,CAAC;IACjD,YAAY,EAAE,mBAAmB,CAAC;CACnC,CAAC,CAUD"}
1
+ {"version":3,"file":"fetch-pipeline.d.ts","sourceRoot":"","sources":["../../src/lib/fetch-pipeline.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,KAAK,uBAAuB,EAAE,MAAM,uBAAuB,CAAC;AAerE,OAAO,EAEL,gBAAgB,EAEhB,UAAU,EACX,MAAM,YAAY,CAAC;AAEpB,OAAO,EAAE,gBAAgB,EAAE,UAAU,EAAE,CAAC;AAExC,eAAO,MAAM,iBAAiB,mBAAmB,CAAC;AAClD,MAAM,WAAW,mBAAmB;IAClC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,OAAO,CAAC;CACrB;AAuED,wBAAgB,sBAAsB,CACpC,OAAO,EAAE,MAAM,EACf,MAAM,EAAE,MAAM,GACb,MAAM,CAkBR;AAgBD,UAAU,oBAAoB,CAAC,CAAC;IAC9B,GAAG,EAAE,MAAM,CAAC;IACZ,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,CAAC;IAC7C,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,gBAAgB,KAAK,IAAI,CAAC;IAC5C,SAAS,EAAE,CACT,KAAK,EAAE;QAAE,MAAM,EAAE,UAAU,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,OAAO,CAAA;KAAE,EACpE,GAAG,EAAE,MAAM,KACR,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IACpB,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,MAAM,CAAC;IAClC,WAAW,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,CAAC,GAAG,SAAS,CAAC;CACjD;AACD,MAAM,WAAW,cAAc,CAAC,CAAC;IAC/B,IAAI,EAAE,CAAC,CAAC;IACR,SAAS,EAAE,OAAO,CAAC;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC1B;AACD,MAAM,MAAM,gBAAgB,GACxB,aAAa,GACb,aAAa,GACb,WAAW,GACX,eAAe,GACf,cAAc,GACd,gBAAgB,GAChB,iBAAiB,GACjB,gBAAgB,GAChB,iBAAiB,CAAC;AAiItB,wBAAsB,oBAAoB,CAAC,CAAC,EAC1C,OAAO,EAAE,oBAAoB,CAAC,CAAC,CAAC,GAC/B,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAoE5B;AAED,MAAM,MAAM,sBAAsB,GAAG,uBAAuB,GAAG;IAC7D,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B,CAAC;AACF,wBAAgB,yBAAyB,CACvC,MAAM,EAAE,MAAM,GACb,sBAAsB,GAAG,SAAS,CAqBpC;AACD,eAAO,MAAM,iBAAiB,GAC5B,OAAO;IAAE,MAAM,EAAE,UAAU,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,OAAO,CAAA;CAAE,EACpE,KAAK,MAAM,EACX,SAAS,WAAW,KACnB,OAAO,CAAC,sBAAsB,CAShC,CAAC;AACF,wBAAgB,uBAAuB,CACrC,MAAM,EAAE,sBAAsB,GAC7B,MAAM,CAaR;AAED,UAAU,kBAAkB;IAC1B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,CAAC;IAC9B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,CAAC;IACtD,QAAQ,CAAC,YAAY,CAAC,EAAE,OAAO,CAAC;IAChC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,gBAAgB,KAAK,IAAI,CAAC;IACrD,QAAQ,CAAC,SAAS,EAAE,CAClB,KAAK,EAAE;QAAE,MAAM,EAAE,UAAU,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,OAAO,CAAA;KAAE,EACpE,aAAa,EAAE,MAAM,KAClB,sBAAsB,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAAC;IAC9D,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,sBAAsB,KAAK,MAAM,CAAC;IAChE,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,sBAAsB,GAAG,SAAS,CAAC;CAC/E;AACD,UAAU,eAAe;IACvB,QAAQ,CAAC,oBAAoB,CAAC,EAAE,OAAO,oBAAoB,CAAC;CAC7D;AAiBD,wBAAsB,kBAAkB,CACtC,OAAO,EAAE,kBAAkB,EAC3B,IAAI,GAAE,eAAoB,GACzB,OAAO,CAAC;IACT,QAAQ,EAAE,cAAc,CAAC,sBAAsB,CAAC,CAAC;IACjD,YAAY,EAAE,mBAAmB,CAAC;CACnC,CAAC,CAUD"}
@@ -3,33 +3,8 @@ import { transformBufferToMarkdown } from '../transform/transform.js';
3
3
  import {} from '../transform/types.js';
4
4
  import { config, createCacheKey, get, isEnabled, logDebug, logWarn, set, } from './core.js';
5
5
  import { fetchNormalizedUrlBuffer, normalizeUrl, transformToRawUrl, } from './http.js';
6
- import { getErrorMessage, isObject } from './utils.js';
7
- function asRecord(value) {
8
- return isObject(value) ? value : undefined;
9
- }
10
- function readUnknown(obj, key) {
11
- const record = asRecord(obj);
12
- return record ? record[key] : undefined;
13
- }
14
- function readString(obj, key) {
15
- const value = readUnknown(obj, key);
16
- return typeof value === 'string' ? value : undefined;
17
- }
18
- export function readNestedRecord(obj, keys) {
19
- let current = obj;
20
- for (const key of keys) {
21
- current = readUnknown(current, key);
22
- if (current === undefined)
23
- return undefined;
24
- }
25
- return asRecord(current);
26
- }
27
- export function withSignal(signal) {
28
- return signal === undefined ? {} : { signal };
29
- }
30
- /* -------------------------------------------------------------------------------------------------
31
- * Inline content truncation
32
- * ------------------------------------------------------------------------------------------------- */
6
+ import { getErrorMessage, readNestedRecord, readString, withSignal, } from './utils.js';
7
+ export { readNestedRecord, withSignal };
33
8
  export const TRUNCATION_MARKER = '...[truncated]';
34
9
  function getOpenCodeFence(content) {
35
10
  const FENCE_PATTERN = /^([ \t]*)(`{3,}|~{3,})/gm;
@@ -106,27 +81,17 @@ export function appendTruncationMarker(content, marker) {
106
81
  }
107
82
  return `${contentWithFence}${marker}`;
108
83
  }
109
- class InlineContentLimiter {
110
- apply(content) {
111
- const contentSize = content.length;
112
- const inlineLimit = config.constants.maxInlineContentChars;
113
- if (isWithinInlineLimit(contentSize, inlineLimit)) {
114
- return { content, contentSize };
115
- }
116
- const truncatedContent = truncateWithMarker(content, inlineLimit, TRUNCATION_MARKER);
117
- return {
118
- content: truncatedContent,
119
- contentSize,
120
- truncated: true,
121
- };
122
- }
123
- }
124
- function isWithinInlineLimit(contentSize, inlineLimit) {
125
- return inlineLimit <= 0 || contentSize <= inlineLimit;
126
- }
127
- const inlineLimiter = new InlineContentLimiter();
128
84
  function applyInlineContentLimit(content) {
129
- return inlineLimiter.apply(content);
85
+ const contentSize = content.length;
86
+ const inlineLimit = config.constants.maxInlineContentChars;
87
+ if (inlineLimit <= 0 || contentSize <= inlineLimit) {
88
+ return { content, contentSize };
89
+ }
90
+ return {
91
+ content: truncateWithMarker(content, inlineLimit, TRUNCATION_MARKER),
92
+ contentSize,
93
+ truncated: true,
94
+ };
130
95
  }
131
96
  function resolveNormalizedUrl(url) {
132
97
  const { normalizedUrl: validatedUrl } = normalizeUrl(url);
@@ -145,16 +110,6 @@ function resolveNormalizedUrl(url) {
145
110
  transformed: true,
146
111
  };
147
112
  }
148
- function logRawUrlTransformation(resolvedUrl) {
149
- if (!resolvedUrl.transformed)
150
- return;
151
- logDebug('Using transformed raw content URL', {
152
- original: resolvedUrl.originalUrl,
153
- });
154
- }
155
- function extractTitle(value) {
156
- return readString(value, 'title');
157
- }
158
113
  function logCacheMiss(reason, cacheNamespace, normalizedUrl, error) {
159
114
  const log = reason.startsWith('deserialize') ? logWarn : logDebug;
160
115
  log(`Cache miss due to ${reason}`, {
@@ -163,19 +118,7 @@ function logCacheMiss(reason, cacheNamespace, normalizedUrl, error) {
163
118
  ...(error ? { error: getErrorMessage(error) } : {}),
164
119
  });
165
120
  }
166
- function createCacheHitResult(params) {
167
- const finalUrl = params.cachedUrl !== params.normalizedUrl ? params.cachedUrl : undefined;
168
- return {
169
- data: params.data,
170
- fromCache: true,
171
- url: params.normalizedUrl,
172
- ...(finalUrl ? { finalUrl } : {}),
173
- fetchedAt: params.fetchedAt,
174
- cacheKey: params.cacheKey,
175
- };
176
- }
177
- function attemptCacheRetrieval(params) {
178
- const { cacheKey, deserialize, cacheNamespace, normalizedUrl } = params;
121
+ function attemptCacheRetrieval(cacheKey, deserialize, cacheNamespace, normalizedUrl) {
179
122
  if (!cacheKey)
180
123
  return null;
181
124
  const cached = get(cacheKey);
@@ -198,26 +141,25 @@ function attemptCacheRetrieval(params) {
198
141
  return null;
199
142
  }
200
143
  logDebug('Cache hit', { namespace: cacheNamespace, url: normalizedUrl });
201
- return createCacheHitResult({
144
+ const finalUrl = cached.url !== normalizedUrl ? cached.url : undefined;
145
+ return {
202
146
  data,
203
- normalizedUrl,
204
- cachedUrl: cached.url,
147
+ fromCache: true,
148
+ url: normalizedUrl,
149
+ ...(finalUrl ? { finalUrl } : {}),
205
150
  fetchedAt: cached.fetchedAt,
206
151
  cacheKey,
207
- });
152
+ };
208
153
  }
209
- function persistCache(params) {
210
- const { cacheKey, data, serialize, normalizedUrl, cacheNamespace, force } = params;
211
- if (!cacheKey)
212
- return;
154
+ function persistCacheEntry(cacheKey, data, serialize, normalizedUrl, cacheNamespace) {
213
155
  const serializer = serialize ?? JSON.stringify;
214
- const title = extractTitle(data);
156
+ const title = readString(data, 'title');
215
157
  const metadata = {
216
158
  url: normalizedUrl,
217
159
  ...(title === undefined ? {} : { title }),
218
160
  };
219
161
  try {
220
- set(cacheKey, serializer(data), metadata, force ? { force: true } : undefined);
162
+ set(cacheKey, serializer(data), metadata);
221
163
  }
222
164
  catch (error) {
223
165
  logWarn('Failed to persist cache entry', {
@@ -227,53 +169,29 @@ function persistCache(params) {
227
169
  });
228
170
  }
229
171
  }
230
- function buildCacheWriteTargets(params) {
231
- const { primaryCacheKey, cacheNamespace, cacheVary, requestedUrl, finalUrl } = params;
232
- const targets = [];
172
+ function persistAllCacheTargets(primaryCacheKey, data, serialize, cacheNamespace, cacheVary, requestedUrl, finalUrl) {
233
173
  if (primaryCacheKey) {
234
- targets.push({
235
- cacheKey: primaryCacheKey,
236
- normalizedUrl: finalUrl ?? requestedUrl,
237
- });
238
- }
239
- if (!finalUrl || finalUrl === requestedUrl) {
240
- return targets;
174
+ persistCacheEntry(primaryCacheKey, data, serialize, finalUrl ?? requestedUrl, cacheNamespace);
241
175
  }
176
+ if (!finalUrl || finalUrl === requestedUrl)
177
+ return;
242
178
  const finalCacheKey = createCacheKey(cacheNamespace, finalUrl, cacheVary);
243
- if (!finalCacheKey || finalCacheKey === primaryCacheKey) {
244
- return targets;
245
- }
246
- targets.push({
247
- cacheKey: finalCacheKey,
248
- normalizedUrl: finalUrl,
249
- });
250
- return targets;
251
- }
252
- function persistCacheTargets(params) {
253
- const { targets, data, serialize, cacheNamespace } = params;
254
- for (const target of targets) {
255
- persistCache({
256
- cacheKey: target.cacheKey,
257
- data,
258
- serialize,
259
- normalizedUrl: target.normalizedUrl,
260
- cacheNamespace,
261
- });
262
- }
179
+ if (!finalCacheKey || finalCacheKey === primaryCacheKey)
180
+ return;
181
+ persistCacheEntry(finalCacheKey, data, serialize, finalUrl, cacheNamespace);
263
182
  }
264
183
  export async function executeFetchPipeline(options) {
265
184
  options.onStage?.('resolve_url');
266
185
  const resolvedUrl = resolveNormalizedUrl(options.url);
267
- logRawUrlTransformation(resolvedUrl);
186
+ if (resolvedUrl.transformed) {
187
+ logDebug('Using transformed raw content URL', {
188
+ original: resolvedUrl.originalUrl,
189
+ });
190
+ }
268
191
  const cacheKey = createCacheKey(options.cacheNamespace, resolvedUrl.normalizedUrl, options.cacheVary);
269
192
  if (!options.forceRefresh) {
270
193
  options.onStage?.('check_cache');
271
- const cachedResult = attemptCacheRetrieval({
272
- cacheKey,
273
- deserialize: options.deserialize,
274
- cacheNamespace: options.cacheNamespace,
275
- normalizedUrl: resolvedUrl.normalizedUrl,
276
- });
194
+ const cachedResult = attemptCacheRetrieval(cacheKey, options.deserialize, options.cacheNamespace, resolvedUrl.normalizedUrl);
277
195
  if (cachedResult) {
278
196
  options.onStage?.('cache_hit');
279
197
  options.onStage?.('cache_restore');
@@ -289,18 +207,7 @@ export async function executeFetchPipeline(options) {
289
207
  options.onStage?.('transform_start');
290
208
  const data = await options.transform({ buffer, encoding, ...(truncated ? { truncated: true } : {}) }, transformUrl);
291
209
  if (isEnabled()) {
292
- persistCacheTargets({
293
- targets: buildCacheWriteTargets({
294
- primaryCacheKey: cacheKey,
295
- cacheNamespace: options.cacheNamespace,
296
- cacheVary: options.cacheVary,
297
- requestedUrl: resolvedUrl.normalizedUrl,
298
- finalUrl,
299
- }),
300
- data,
301
- serialize: options.serialize,
302
- cacheNamespace: options.cacheNamespace,
303
- });
210
+ persistAllCacheTargets(cacheKey, data, options.serialize, options.cacheNamespace, options.cacheVary, resolvedUrl.normalizedUrl, finalUrl);
304
211
  }
305
212
  return {
306
213
  data,
@@ -316,7 +223,7 @@ export function parseCachedMarkdownResult(cached) {
316
223
  const payload = parseCachedPayload(cached);
317
224
  if (!payload)
318
225
  return undefined;
319
- const markdown = payload.markdown ?? payload.content;
226
+ const { markdown } = payload;
320
227
  if (typeof markdown !== 'string')
321
228
  return undefined;
322
229
  const metadata = normalizeExtractedMetadata(payload.metadata);
@@ -355,17 +262,24 @@ export function serializeMarkdownResult(result) {
355
262
  return JSON.stringify(payload);
356
263
  }
357
264
  function buildSharedFetchPipelineOptions(options) {
358
- return {
265
+ const opts = {
359
266
  url: options.url,
360
267
  cacheNamespace: 'markdown',
361
- ...withSignal(options.signal),
362
- ...(options.cacheVary ? { cacheVary: options.cacheVary } : {}),
363
- ...(options.forceRefresh ? { forceRefresh: true } : {}),
364
- ...(options.onStage ? { onStage: options.onStage } : {}),
365
268
  transform: options.transform,
366
- ...(options.serialize ? { serialize: options.serialize } : {}),
367
- ...(options.deserialize ? { deserialize: options.deserialize } : {}),
368
269
  };
270
+ if (options.signal !== undefined)
271
+ opts.signal = options.signal;
272
+ if (options.cacheVary !== undefined)
273
+ opts.cacheVary = options.cacheVary;
274
+ if (options.forceRefresh)
275
+ opts.forceRefresh = true;
276
+ if (options.onStage)
277
+ opts.onStage = options.onStage;
278
+ if (options.serialize)
279
+ opts.serialize = options.serialize;
280
+ if (options.deserialize)
281
+ opts.deserialize = options.deserialize;
282
+ return opts;
369
283
  }
370
284
  export async function performSharedFetch(options, deps = {}) {
371
285
  const executePipeline = deps.executeFetchPipeline ?? executeFetchPipeline;
@@ -1 +1 @@
1
- {"version":3,"file":"http.d.ts","sourceRoot":"","sources":["../../src/lib/http.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,KAAK,cAAc,EAAE,MAAM,WAAW,CAAC;AAWhD,OAAO,EAAE,KAAK,EAAmB,MAAM,QAAQ,CAAC;AAchD,OAAO,EAOL,KAAK,eAAe,EAGrB,MAAM,UAAU,CAAC;AAiElB,wBAAgB,oBAAoB,CAClC,GAAG,EAAE,MAAM,EACX,KAAK,CAAC,EAAE,MAAM,EACd,YAAY,CAAC,EAAE,MAAM,EACrB,SAAS,SAAQ,GAChB,MAAM,CAIR;AAkBD,wBAAgB,cAAc,CAC5B,GAAG,EAAE,cAAc,EACnB,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,MAAM,GACX,IAAI,CA6CN;AAmoCD,UAAU,qBAAqB;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAuJD,UAAU,YAAY;IACpB,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAsMD,wBAAgB,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAE/C;AACD,wBAAgB,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG;IAC/C,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;CAClB,CAEA;AACD,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAEjE;AACD,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,eAAe,CAE9D;AACD,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAExD;AACD,wBAAgB,mBAAmB,CACjC,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,MAAM,GACb,qBAAqB,CAEvB;AACD,wBAAgB,mBAAmB,CACjC,OAAO,EAAE,qBAAqB,EAC9B,QAAQ,EAAE,QAAQ,EAClB,WAAW,CAAC,EAAE,MAAM,GACnB,IAAI,CAEN;AACD,wBAAgB,gBAAgB,CAC9B,OAAO,EAAE,qBAAqB,EAC9B,KAAK,EAAE,OAAO,EACd,MAAM,CAAC,EAAE,MAAM,GACd,IAAI,CAEN;AACD,wBAAsB,kBAAkB,CACtC,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,WAAW,EACjB,YAAY,EAAE,MAAM,GACnB,OAAO,CAAC;IAAE,QAAQ,EAAE,QAAQ,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,KAAK,CAAA;CAAE,CAAC,CAE7D;AACD,wBAAsB,gBAAgB,CACpC,QAAQ,EAAE,QAAQ,EAClB,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,MAAM,EAChB,MAAM,CAAC,EAAE,WAAW,EACpB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC,CAUzC;AACD,wBAAsB,kBAAkB,CACtC,aAAa,EAAE,MAAM,EACrB,OAAO,CAAC,EAAE,YAAY,GACrB,OAAO,CAAC,MAAM,CAAC,CAEjB;AACD,wBAAsB,wBAAwB,CAC5C,aAAa,EAAE,MAAM,EACrB,OAAO,CAAC,EAAE,YAAY,GACrB,OAAO,CAAC;IACT,MAAM,EAAE,UAAU,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,OAAO,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC,CAED"}
1
+ {"version":3,"file":"http.d.ts","sourceRoot":"","sources":["../../src/lib/http.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,KAAK,cAAc,EAAE,MAAM,WAAW,CAAC;AAWhD,OAAO,EAAE,KAAK,EAAmB,MAAM,QAAQ,CAAC;AAchD,OAAO,EAOL,KAAK,eAAe,EAGrB,MAAM,UAAU,CAAC;AAqElB,wBAAgB,oBAAoB,CAClC,GAAG,EAAE,MAAM,EACX,KAAK,CAAC,EAAE,MAAM,EACd,YAAY,CAAC,EAAE,MAAM,EACrB,SAAS,SAAQ,GAChB,MAAM,CAIR;AAkBD,wBAAgB,cAAc,CAC5B,GAAG,EAAE,cAAc,EACnB,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,MAAM,GACX,IAAI,CA6CN;AAktCD,UAAU,qBAAqB;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAgKD,UAAU,YAAY;IACpB,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAgOD,wBAAgB,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAE/C;AACD,wBAAgB,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG;IAC/C,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;CAClB,CAEA;AACD,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAEjE;AACD,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,eAAe,CAE9D;AACD,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAExD;AACD,wBAAgB,mBAAmB,CACjC,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,MAAM,GACb,qBAAqB,CAEvB;AACD,wBAAgB,mBAAmB,CACjC,OAAO,EAAE,qBAAqB,EAC9B,QAAQ,EAAE,QAAQ,EAClB,WAAW,CAAC,EAAE,MAAM,GACnB,IAAI,CAEN;AACD,wBAAgB,gBAAgB,CAC9B,OAAO,EAAE,qBAAqB,EAC9B,KAAK,EAAE,OAAO,EACd,MAAM,CAAC,EAAE,MAAM,GACd,IAAI,CAEN;AACD,wBAAsB,kBAAkB,CACtC,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,WAAW,EACjB,YAAY,EAAE,MAAM,GACnB,OAAO,CAAC;IAAE,QAAQ,EAAE,QAAQ,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,KAAK,CAAA;CAAE,CAAC,CAE7D;AACD,wBAAsB,gBAAgB,CACpC,QAAQ,EAAE,QAAQ,EAClB,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,MAAM,EAChB,MAAM,CAAC,EAAE,WAAW,EACpB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC,CAUzC;AACD,wBAAsB,kBAAkB,CACtC,aAAa,EAAE,MAAM,EACrB,OAAO,CAAC,EAAE,YAAY,GACrB,OAAO,CAAC,MAAM,CAAC,CAEjB;AACD,wBAAsB,wBAAwB,CAC5C,aAAa,EAAE,MAAM,EACrB,OAAO,CAAC,EAAE,YAAY,GACrB,OAAO,CAAC;IACT,MAAM,EAAE,UAAU,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,OAAO,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC,CAED"}