@j0hanz/fetch-url-mcp 1.3.1 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (205) hide show
  1. package/README.md +24 -21
  2. package/dist/cli.d.ts +3 -3
  3. package/dist/cli.js +15 -8
  4. package/dist/http/auth.d.ts +6 -6
  5. package/dist/http/auth.js +78 -23
  6. package/dist/http/health.d.ts +1 -2
  7. package/dist/http/health.js +7 -18
  8. package/dist/http/helpers.d.ts +3 -11
  9. package/dist/http/helpers.js +28 -26
  10. package/dist/http/native.d.ts +0 -1
  11. package/dist/http/native.js +63 -41
  12. package/dist/http/rate-limit.d.ts +2 -2
  13. package/dist/http/rate-limit.js +11 -16
  14. package/dist/index.d.ts +0 -1
  15. package/dist/index.js +17 -20
  16. package/dist/{markdown-cleanup.d.ts → lib/content.d.ts} +4 -2
  17. package/dist/lib/content.js +1356 -0
  18. package/dist/lib/core.d.ts +253 -0
  19. package/dist/lib/core.js +1228 -0
  20. package/dist/{tool-pipeline.d.ts → lib/fetch-pipeline.d.ts} +1 -3
  21. package/dist/{tool-pipeline.js → lib/fetch-pipeline.js} +18 -44
  22. package/dist/{fetch.d.ts → lib/http.d.ts} +7 -9
  23. package/dist/{fetch.js → lib/http.js} +721 -1004
  24. package/dist/lib/mcp-tools.d.ts +28 -0
  25. package/dist/lib/mcp-tools.js +107 -0
  26. package/dist/{tool-progress.d.ts → lib/progress.d.ts} +0 -2
  27. package/dist/{tool-progress.js → lib/progress.js} +9 -14
  28. package/dist/lib/task-handlers.d.ts +5 -0
  29. package/dist/{mcp.js → lib/task-handlers.js} +95 -31
  30. package/dist/lib/url.d.ts +70 -0
  31. package/dist/lib/url.js +686 -0
  32. package/dist/lib/utils.d.ts +58 -0
  33. package/dist/lib/utils.js +304 -0
  34. package/dist/{prompts.d.ts → prompts/index.d.ts} +0 -1
  35. package/dist/{prompts.js → prompts/index.js} +1 -2
  36. package/dist/{resources.d.ts → resources/index.d.ts} +0 -1
  37. package/dist/{resources.js → resources/index.js} +87 -64
  38. package/dist/{instructions.d.ts → resources/instructions.d.ts} +0 -1
  39. package/dist/{instructions.js → resources/instructions.js} +5 -3
  40. package/dist/schemas/inputs.d.ts +7 -0
  41. package/dist/schemas/inputs.js +24 -0
  42. package/dist/schemas/outputs.d.ts +23 -0
  43. package/dist/schemas/outputs.js +77 -0
  44. package/dist/server.d.ts +0 -1
  45. package/dist/server.js +26 -25
  46. package/dist/tasks/execution.d.ts +0 -1
  47. package/dist/tasks/execution.js +106 -70
  48. package/dist/tasks/manager.d.ts +11 -3
  49. package/dist/tasks/manager.js +97 -73
  50. package/dist/tasks/owner.d.ts +3 -3
  51. package/dist/tasks/owner.js +2 -2
  52. package/dist/tasks/tool-registry.d.ts +11 -0
  53. package/dist/tasks/tool-registry.js +13 -0
  54. package/dist/tools/fetch-url.d.ts +28 -0
  55. package/dist/{tools.js → tools/fetch-url.js} +95 -147
  56. package/dist/tools/index.d.ts +2 -0
  57. package/dist/tools/index.js +4 -0
  58. package/dist/transform/html-translators.d.ts +1 -0
  59. package/dist/transform/html-translators.js +454 -0
  60. package/dist/transform/metadata.d.ts +4 -0
  61. package/dist/transform/metadata.js +183 -0
  62. package/dist/transform/transform.d.ts +0 -1
  63. package/dist/transform/transform.js +44 -679
  64. package/dist/transform/types.d.ts +9 -12
  65. package/dist/transform/types.js +0 -1
  66. package/dist/transform/worker-pool.d.ts +0 -1
  67. package/dist/transform/worker-pool.js +7 -16
  68. package/dist/transform/workers/shared.d.ts +7 -0
  69. package/dist/transform/workers/shared.js +130 -0
  70. package/dist/transform/workers/transform-child.d.ts +0 -1
  71. package/dist/transform/workers/transform-child.js +5 -135
  72. package/dist/transform/workers/transform-worker.d.ts +0 -1
  73. package/dist/transform/workers/transform-worker.js +7 -128
  74. package/package.json +11 -7
  75. package/dist/cache.d.ts +0 -54
  76. package/dist/cache.d.ts.map +0 -1
  77. package/dist/cache.js +0 -261
  78. package/dist/cache.js.map +0 -1
  79. package/dist/cli.d.ts.map +0 -1
  80. package/dist/cli.js.map +0 -1
  81. package/dist/config.d.ts +0 -141
  82. package/dist/config.d.ts.map +0 -1
  83. package/dist/config.js +0 -473
  84. package/dist/config.js.map +0 -1
  85. package/dist/crypto.d.ts +0 -4
  86. package/dist/crypto.d.ts.map +0 -1
  87. package/dist/crypto.js +0 -56
  88. package/dist/crypto.js.map +0 -1
  89. package/dist/dom-noise-removal.d.ts +0 -2
  90. package/dist/dom-noise-removal.d.ts.map +0 -1
  91. package/dist/dom-noise-removal.js +0 -494
  92. package/dist/dom-noise-removal.js.map +0 -1
  93. package/dist/download.d.ts +0 -4
  94. package/dist/download.d.ts.map +0 -1
  95. package/dist/download.js +0 -106
  96. package/dist/download.js.map +0 -1
  97. package/dist/errors.d.ts +0 -11
  98. package/dist/errors.d.ts.map +0 -1
  99. package/dist/errors.js +0 -65
  100. package/dist/errors.js.map +0 -1
  101. package/dist/examples/mcp-fetch-url-client.js +0 -329
  102. package/dist/examples/mcp-fetch-url-client.js.map +0 -1
  103. package/dist/fetch-content.d.ts +0 -5
  104. package/dist/fetch-content.d.ts.map +0 -1
  105. package/dist/fetch-content.js +0 -164
  106. package/dist/fetch-content.js.map +0 -1
  107. package/dist/fetch-stream.d.ts +0 -5
  108. package/dist/fetch-stream.d.ts.map +0 -1
  109. package/dist/fetch-stream.js +0 -29
  110. package/dist/fetch-stream.js.map +0 -1
  111. package/dist/fetch.d.ts.map +0 -1
  112. package/dist/fetch.js.map +0 -1
  113. package/dist/host-normalization.d.ts +0 -2
  114. package/dist/host-normalization.d.ts.map +0 -1
  115. package/dist/host-normalization.js +0 -91
  116. package/dist/host-normalization.js.map +0 -1
  117. package/dist/http/auth.d.ts.map +0 -1
  118. package/dist/http/auth.js.map +0 -1
  119. package/dist/http/health.d.ts.map +0 -1
  120. package/dist/http/health.js.map +0 -1
  121. package/dist/http/helpers.d.ts.map +0 -1
  122. package/dist/http/helpers.js.map +0 -1
  123. package/dist/http/native.d.ts.map +0 -1
  124. package/dist/http/native.js.map +0 -1
  125. package/dist/http/rate-limit.d.ts.map +0 -1
  126. package/dist/http/rate-limit.js.map +0 -1
  127. package/dist/index.d.ts.map +0 -1
  128. package/dist/index.js.map +0 -1
  129. package/dist/instructions.d.ts.map +0 -1
  130. package/dist/instructions.js.map +0 -1
  131. package/dist/ip-blocklist.d.ts +0 -9
  132. package/dist/ip-blocklist.d.ts.map +0 -1
  133. package/dist/ip-blocklist.js +0 -79
  134. package/dist/ip-blocklist.js.map +0 -1
  135. package/dist/json.d.ts +0 -2
  136. package/dist/json.d.ts.map +0 -1
  137. package/dist/json.js +0 -45
  138. package/dist/json.js.map +0 -1
  139. package/dist/language-detection.d.ts +0 -3
  140. package/dist/language-detection.d.ts.map +0 -1
  141. package/dist/language-detection.js +0 -355
  142. package/dist/language-detection.js.map +0 -1
  143. package/dist/markdown-cleanup.d.ts.map +0 -1
  144. package/dist/markdown-cleanup.js +0 -534
  145. package/dist/markdown-cleanup.js.map +0 -1
  146. package/dist/mcp-validator.d.ts +0 -17
  147. package/dist/mcp-validator.d.ts.map +0 -1
  148. package/dist/mcp-validator.js +0 -45
  149. package/dist/mcp-validator.js.map +0 -1
  150. package/dist/mcp.d.ts +0 -4
  151. package/dist/mcp.d.ts.map +0 -1
  152. package/dist/mcp.js.map +0 -1
  153. package/dist/observability.d.ts +0 -23
  154. package/dist/observability.d.ts.map +0 -1
  155. package/dist/observability.js +0 -238
  156. package/dist/observability.js.map +0 -1
  157. package/dist/prompts.d.ts.map +0 -1
  158. package/dist/prompts.js.map +0 -1
  159. package/dist/resources.d.ts.map +0 -1
  160. package/dist/resources.js.map +0 -1
  161. package/dist/server-tuning.d.ts +0 -15
  162. package/dist/server-tuning.d.ts.map +0 -1
  163. package/dist/server-tuning.js +0 -49
  164. package/dist/server-tuning.js.map +0 -1
  165. package/dist/server.d.ts.map +0 -1
  166. package/dist/server.js.map +0 -1
  167. package/dist/session.d.ts +0 -42
  168. package/dist/session.d.ts.map +0 -1
  169. package/dist/session.js +0 -255
  170. package/dist/session.js.map +0 -1
  171. package/dist/tasks/execution.d.ts.map +0 -1
  172. package/dist/tasks/execution.js.map +0 -1
  173. package/dist/tasks/manager.d.ts.map +0 -1
  174. package/dist/tasks/manager.js.map +0 -1
  175. package/dist/tasks/owner.d.ts.map +0 -1
  176. package/dist/tasks/owner.js.map +0 -1
  177. package/dist/timer-utils.d.ts +0 -6
  178. package/dist/timer-utils.d.ts.map +0 -1
  179. package/dist/timer-utils.js +0 -27
  180. package/dist/timer-utils.js.map +0 -1
  181. package/dist/tool-errors.d.ts +0 -12
  182. package/dist/tool-errors.d.ts.map +0 -1
  183. package/dist/tool-errors.js +0 -55
  184. package/dist/tool-errors.js.map +0 -1
  185. package/dist/tool-pipeline.d.ts.map +0 -1
  186. package/dist/tool-pipeline.js.map +0 -1
  187. package/dist/tool-progress.d.ts.map +0 -1
  188. package/dist/tool-progress.js.map +0 -1
  189. package/dist/tools.d.ts +0 -54
  190. package/dist/tools.d.ts.map +0 -1
  191. package/dist/tools.js.map +0 -1
  192. package/dist/transform/transform.d.ts.map +0 -1
  193. package/dist/transform/transform.js.map +0 -1
  194. package/dist/transform/types.d.ts.map +0 -1
  195. package/dist/transform/types.js.map +0 -1
  196. package/dist/transform/worker-pool.d.ts.map +0 -1
  197. package/dist/transform/worker-pool.js.map +0 -1
  198. package/dist/transform/workers/transform-child.d.ts.map +0 -1
  199. package/dist/transform/workers/transform-child.js.map +0 -1
  200. package/dist/transform/workers/transform-worker.d.ts.map +0 -1
  201. package/dist/transform/workers/transform-worker.js.map +0 -1
  202. package/dist/type-guards.d.ts +0 -16
  203. package/dist/type-guards.d.ts.map +0 -1
  204. package/dist/type-guards.js +0 -13
  205. package/dist/type-guards.js.map +0 -1
package/dist/errors.js DELETED
@@ -1,65 +0,0 @@
1
- import { inspect } from 'node:util';
2
- import { isError, isObject } from './type-guards.js';
3
- const DEFAULT_HTTP_STATUS = 502;
4
- export class FetchError extends Error {
5
- url;
6
- statusCode;
7
- code;
8
- details;
9
- constructor(message, url, httpStatus, details = {}, options) {
10
- super(message, options);
11
- this.url = url;
12
- this.name = 'FetchError';
13
- this.statusCode = httpStatus ?? DEFAULT_HTTP_STATUS;
14
- this.code = httpStatus ? `HTTP_${httpStatus}` : 'FETCH_ERROR';
15
- this.details = Object.freeze({ url, httpStatus, ...details });
16
- Error.captureStackTrace(this, this.constructor);
17
- }
18
- }
19
- export function getErrorMessage(error) {
20
- if (isError(error))
21
- return error.message;
22
- if (isNonEmptyString(error))
23
- return error;
24
- if (isErrorWithMessage(error))
25
- return error.message;
26
- return formatUnknownError(error);
27
- }
28
- function isNonEmptyString(value) {
29
- return typeof value === 'string' && value.length > 0;
30
- }
31
- function isErrorWithMessage(error) {
32
- if (!isObject(error))
33
- return false;
34
- const { message } = error;
35
- return typeof message === 'string' && message.length > 0;
36
- }
37
- function formatUnknownError(error) {
38
- if (error === null || error === undefined)
39
- return 'Unknown error';
40
- try {
41
- return inspect(error, {
42
- depth: 2,
43
- maxStringLength: 200,
44
- breakLength: Infinity,
45
- compact: true,
46
- colors: false,
47
- });
48
- }
49
- catch {
50
- return 'Unknown error';
51
- }
52
- }
53
- export function createErrorWithCode(message, code, options) {
54
- const error = new Error(message, options);
55
- return Object.assign(error, { code });
56
- }
57
- export function isSystemError(error) {
58
- if (!isError(error))
59
- return false;
60
- if (!('code' in error))
61
- return false;
62
- const { code } = error;
63
- return typeof code === 'string';
64
- }
65
- //# sourceMappingURL=errors.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"errors.js","sourceRoot":"","sources":["../src/errors.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAErD,MAAM,mBAAmB,GAAG,GAAG,CAAC;AAEhC,MAAM,OAAO,UAAW,SAAQ,KAAK;IAOxB;IANF,UAAU,CAAS;IACnB,IAAI,CAAS;IACb,OAAO,CAAoC;IAEpD,YACE,OAAe,EACN,GAAW,EACpB,UAAmB,EACnB,UAAmC,EAAE,EACrC,OAAsB;QAEtB,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QALf,QAAG,GAAH,GAAG,CAAQ;QAMpB,IAAI,CAAC,IAAI,GAAG,YAAY,CAAC;QACzB,IAAI,CAAC,UAAU,GAAG,UAAU,IAAI,mBAAmB,CAAC;QACpD,IAAI,CAAC,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC,QAAQ,UAAU,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC;QAC9D,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,GAAG,OAAO,EAAE,CAAC,CAAC;QAC9D,KAAK,CAAC,iBAAiB,CAAC,IAAI,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;IAClD,CAAC;CACF;AAED,MAAM,UAAU,eAAe,CAAC,KAAc;IAC5C,IAAI,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC,OAAO,CAAC;IACzC,IAAI,gBAAgB,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAC1C,IAAI,kBAAkB,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC,OAAO,CAAC;IACpD,OAAO,kBAAkB,CAAC,KAAK,CAAC,CAAC;AACnC,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAc;IACtC,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;AACvD,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAc;IACxC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACnC,MAAM,EAAE,OAAO,EAAE,GAAG,KAAK,CAAC;IAC1B,OAAO,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;AAC3D,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAc;IACxC,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,eAAe,CAAC;IAClE,IAAI,CAAC;QACH,OAAO,OAAO,CAAC,KAAK,EAAE;YACpB,KAAK,EAAE,CAAC;YACR,eAAe,EAAE,GAAG;YACpB,WAAW,EAAE,QAAQ;YACrB,OAAO,EAAE,IAAI;YACb,MAAM,EAAE,KAAK;SACd,CAAC,CAAC;IACL,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,eAAe,CAAC;IACzB,CAAC;AACH,CAAC;AAED,MAAM,UAAU,mBAAmB,CACjC,OAAe,EACf,IAAY,EACZ,OAAsB;IAEtB,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC1C,OAAO,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;AACxC,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,KAAc;IAC1C,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAClC,IAAI,CAAC,CAAC,MAAM,IAAI,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACrC,MAAM,EAAE,IAAI,EAAE,GAAG,KAA2B,CAAC;IAC7C,OAAO,OAAO,IAAI,KAAK,QAAQ,CAAC;AAClC,CAAC"}
@@ -1,329 +0,0 @@
1
- import { Client } from '@modelcontextprotocol/sdk/client';
2
- import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js';
3
- import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js';
4
- import { access, writeFile } from 'node:fs/promises';
5
- import path from 'node:path';
6
- import process from 'node:process';
7
- import { fileURLToPath } from 'node:url';
8
- import { parseArgs } from 'node:util';
9
- const __filename = fileURLToPath(import.meta.url);
10
- const __dirname = path.dirname(__filename);
11
- function printUsage() {
12
- const usage = `
13
- Usage:
14
- node dist/examples/mcp-fetch-url-client.js <url> [options]
15
-
16
- Options:
17
- --http <url> Connect via Streamable HTTP (e.g. http://localhost:3000/mcp)
18
- --task Use task-based execution with streamed status updates
19
- --task-ttl <ms> Task TTL in milliseconds (optional)
20
- --task-poll <ms> Task poll interval in milliseconds (optional)
21
- --no-noise Skip noise removal
22
- --force Force refresh (bypass cache)
23
- --max-inline <n> Max inline chars before truncation
24
- --full If truncated, read cached resource for full markdown
25
- --out <path> Write markdown to file instead of stdout
26
- --json Print full structured JSON instead of markdown
27
- --cmd <executable> Stdio: command to spawn (default: node)
28
- --server <path> Stdio: server entry (default: dist/index.js)
29
- --cwd <path> Stdio: working directory for server (default: repo root)
30
- --env KEY=VALUE Stdio: add/override environment variable (repeatable)
31
- -h, --help Show help
32
- `;
33
- process.stderr.write(usage);
34
- }
35
- async function findRepoRoot(startDir) {
36
- let current = startDir;
37
- while (true) {
38
- const candidate = path.join(current, 'package.json');
39
- try {
40
- await access(candidate);
41
- return current;
42
- }
43
- catch {
44
- const parent = path.dirname(current);
45
- if (parent === current) {
46
- return startDir;
47
- }
48
- current = parent;
49
- }
50
- }
51
- }
52
- function parseEnvOverrides(values) {
53
- const env = {};
54
- if (!values) {
55
- return env;
56
- }
57
- for (const item of values) {
58
- const index = item.indexOf('=');
59
- if (index <= 0) {
60
- throw new Error(`Invalid --env value: ${item}`);
61
- }
62
- const key = item.slice(0, index).trim();
63
- const value = item.slice(index + 1);
64
- if (!key) {
65
- throw new Error(`Invalid --env key in: ${item}`);
66
- }
67
- env[key] = value;
68
- }
69
- return env;
70
- }
71
- function buildInheritedEnv() {
72
- const env = {};
73
- for (const [key, value] of Object.entries(process.env)) {
74
- if (typeof value === 'string') {
75
- env[key] = value;
76
- }
77
- }
78
- return env;
79
- }
80
- function pickTextResource(contents) {
81
- for (const entry of contents) {
82
- const text = entry['text'];
83
- if (typeof text === 'string') {
84
- return text;
85
- }
86
- }
87
- return null;
88
- }
89
- function formatProgress(progress) {
90
- const { message, total } = progress;
91
- if (typeof total === 'number' && total > 0) {
92
- const percent = Math.round((progress.progress / total) * 100);
93
- return `${percent}%${message ? ` ${message}` : ''}`;
94
- }
95
- return message ? `${progress.progress} ${message}` : `${progress.progress}`;
96
- }
97
- function getStructuredContent(result) {
98
- if (typeof result !== 'object' || result === null) {
99
- return null;
100
- }
101
- const candidate = result;
102
- if (candidate.structuredContent &&
103
- typeof candidate.structuredContent === 'object' &&
104
- !Array.isArray(candidate.structuredContent)) {
105
- return candidate.structuredContent;
106
- }
107
- if (candidate.content &&
108
- Array.isArray(candidate.content) &&
109
- candidate.content[0]?.type === 'text' &&
110
- candidate.content[0].text) {
111
- try {
112
- const parsed = JSON.parse(candidate.content[0].text);
113
- if (typeof parsed === 'object' &&
114
- parsed !== null &&
115
- !Array.isArray(parsed)) {
116
- return parsed;
117
- }
118
- }
119
- catch {
120
- // ignore
121
- }
122
- }
123
- return null;
124
- }
125
- function isToolError(result) {
126
- if (typeof result !== 'object' || result === null) {
127
- return false;
128
- }
129
- const candidate = result;
130
- return candidate.isError === true;
131
- }
132
- function getStringField(structured, key) {
133
- if (!structured) {
134
- return null;
135
- }
136
- const value = structured[key];
137
- return typeof value === 'string' ? value : null;
138
- }
139
- const options = {
140
- help: { type: 'boolean', short: 'h' },
141
- http: { type: 'string' },
142
- task: { type: 'boolean' },
143
- 'task-ttl': { type: 'string' },
144
- 'task-poll': { type: 'string' },
145
- 'no-noise': { type: 'boolean' },
146
- force: { type: 'boolean' },
147
- 'max-inline': { type: 'string' },
148
- full: { type: 'boolean' },
149
- out: { type: 'string' },
150
- json: { type: 'boolean' },
151
- cmd: { type: 'string' },
152
- server: { type: 'string' },
153
- cwd: { type: 'string' },
154
- env: { type: 'string', multiple: true },
155
- };
156
- const { values, positionals } = parseArgs({
157
- allowPositionals: true,
158
- options,
159
- });
160
- if (values.help) {
161
- printUsage();
162
- process.exit(0);
163
- }
164
- function requireUrl(positionals) {
165
- const url = positionals[0];
166
- if (typeof url !== 'string' || url.length === 0) {
167
- printUsage();
168
- throw new Error('Missing URL.');
169
- }
170
- return url;
171
- }
172
- const targetUrl = requireUrl(positionals);
173
- const maxInlineRaw = values['max-inline'];
174
- const maxInlineChars = maxInlineRaw !== undefined ? Number(maxInlineRaw) : undefined;
175
- if (maxInlineRaw !== undefined && Number.isNaN(maxInlineChars)) {
176
- throw new Error(`Invalid --max-inline value: ${maxInlineRaw}`);
177
- }
178
- const taskTtlRaw = values['task-ttl'];
179
- const taskTtl = taskTtlRaw !== undefined ? Number(taskTtlRaw) : undefined;
180
- if (taskTtlRaw !== undefined && Number.isNaN(taskTtl)) {
181
- throw new Error(`Invalid --task-ttl value: ${taskTtlRaw}`);
182
- }
183
- const taskPollRaw = values['task-poll'];
184
- const taskPoll = taskPollRaw !== undefined ? Number(taskPollRaw) : undefined;
185
- if (taskPollRaw !== undefined && Number.isNaN(taskPoll)) {
186
- throw new Error(`Invalid --task-poll value: ${taskPollRaw}`);
187
- }
188
- const onProgress = (progress) => {
189
- process.stderr.write(`[progress] ${formatProgress(progress)}\n`);
190
- };
191
- async function run() {
192
- let transport = null;
193
- const client = new Client({ name: 'fetch-url-mcp-client', version: '0.1.0' }, { capabilities: {} });
194
- try {
195
- if (values.http) {
196
- const endpoint = new URL(values.http);
197
- transport = new StreamableHTTPClientTransport(endpoint);
198
- }
199
- else {
200
- const command = values.cmd ?? process.execPath;
201
- const repoRoot = await findRepoRoot(__dirname);
202
- const serverPath = values.server ?? path.join(repoRoot, 'dist/index.js');
203
- const cwd = values.cwd ?? repoRoot;
204
- try {
205
- await access(serverPath);
206
- }
207
- catch {
208
- throw new Error(`Server entry not found at ${serverPath}. Run \"npm run build\" first or set --server.`);
209
- }
210
- const env = {
211
- ...buildInheritedEnv(),
212
- ...parseEnvOverrides(values.env),
213
- };
214
- transport = new StdioClientTransport({
215
- command,
216
- args: [serverPath, '--stdio'],
217
- cwd,
218
- env,
219
- stderr: 'inherit',
220
- });
221
- }
222
- await client.connect(transport);
223
- const toolArguments = {
224
- url: targetUrl,
225
- skipNoiseRemoval: values['no-noise'] ?? false,
226
- forceRefresh: values.force ?? false,
227
- };
228
- if (typeof maxInlineChars === 'number') {
229
- toolArguments.maxInlineChars = maxInlineChars;
230
- }
231
- const taskOptions = {};
232
- if (typeof taskTtl === 'number') {
233
- taskOptions.ttl = taskTtl;
234
- }
235
- if (typeof taskPoll === 'number') {
236
- taskOptions.pollInterval = taskPoll;
237
- }
238
- const requestOptions = values.task
239
- ? { onprogress: onProgress, task: taskOptions }
240
- : { onprogress: onProgress };
241
- const result = values.task
242
- ? await callToolStream(client, toolArguments, requestOptions)
243
- : await client.callTool({
244
- name: 'fetch-url',
245
- arguments: toolArguments,
246
- }, undefined, requestOptions);
247
- if (isToolError(result)) {
248
- const errorPayload = getStructuredContent(result) ?? {
249
- message: 'Fetch failed',
250
- };
251
- process.stderr.write(`${JSON.stringify(errorPayload, null, 2)}\n`);
252
- process.exitCode = 2;
253
- return;
254
- }
255
- const structured = getStructuredContent(result);
256
- if (values.json) {
257
- const payload = structured ?? result;
258
- process.stdout.write(`${JSON.stringify(payload, null, 2)}\n`);
259
- return;
260
- }
261
- let markdown = getStringField(structured, 'markdown');
262
- if (values.full) {
263
- const cacheResourceUri = getStringField(structured, 'cacheResourceUri');
264
- if (cacheResourceUri) {
265
- const resource = await client.readResource({
266
- uri: cacheResourceUri,
267
- });
268
- const text = pickTextResource(resource.contents);
269
- if (text) {
270
- markdown = text;
271
- }
272
- }
273
- }
274
- if (!markdown) {
275
- throw new Error('No markdown returned from fetch-url.');
276
- }
277
- if (values.out) {
278
- await writeFile(values.out, markdown, 'utf8');
279
- }
280
- else {
281
- process.stdout.write(`${markdown}\n`);
282
- }
283
- }
284
- finally {
285
- if (transport) {
286
- await transport.close();
287
- }
288
- }
289
- }
290
- async function callToolStream(client, toolArguments, requestOptions) {
291
- const stream = client.experimental.tasks.callToolStream({
292
- name: 'fetch-url',
293
- arguments: toolArguments,
294
- }, undefined, requestOptions);
295
- let finalResult = null;
296
- for await (const message of stream) {
297
- if (message.type === 'taskCreated') {
298
- process.stderr.write(`[task] created ${message.task.taskId}\n`);
299
- continue;
300
- }
301
- if (message.type === 'taskStatus') {
302
- const statusMessage = message.task.statusMessage
303
- ? ` ${message.task.statusMessage}`
304
- : '';
305
- process.stderr.write(`[task] ${message.task.status}${statusMessage}\n`);
306
- continue;
307
- }
308
- if (message.type === 'result') {
309
- finalResult = message.result;
310
- continue;
311
- }
312
- if (message.type === 'error') {
313
- throw new Error(message.error.message ?? 'Task failed');
314
- }
315
- }
316
- if (finalResult === null) {
317
- throw new Error('Task stream ended without a result.');
318
- }
319
- return finalResult;
320
- }
321
- try {
322
- await run();
323
- }
324
- catch (error) {
325
- const message = error instanceof Error ? error.message : String(error);
326
- process.stderr.write(`${message}\n`);
327
- process.exitCode = 1;
328
- }
329
- //# sourceMappingURL=mcp-fetch-url-client.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"mcp-fetch-url-client.js","sourceRoot":"","sources":["../../examples/mcp-fetch-url-client.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,kCAAkC,CAAC;AAC1D,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AACjF,OAAO,EAAE,6BAA6B,EAAE,MAAM,oDAAoD,CAAC;AAInG,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,OAAO,MAAM,cAAc,CAAC;AACnC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAQtC,MAAM,UAAU,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAClD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;AAE3C,SAAS,UAAU;IACjB,MAAM,KAAK,GAAG;;;;;;;;;;;;;;;;;;;;CAoBf,CAAC;IACA,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;AAC9B,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,QAAgB;IAC1C,IAAI,OAAO,GAAG,QAAQ,CAAC;IACvB,OAAO,IAAI,EAAE,CAAC;QACZ,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;QACrD,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;YACxB,OAAO,OAAO,CAAC;QACjB,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;YACrC,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;gBACvB,OAAO,QAAQ,CAAC;YAClB,CAAC;YACD,OAAO,GAAG,MAAM,CAAC;QACnB,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB,CACxB,MAA4B;IAE5B,MAAM,GAAG,GAA2B,EAAE,CAAC;IACvC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,GAAG,CAAC;IACb,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAC1B,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,wBAAwB,IAAI,EAAE,CAAC,CAAC;QAClD,CAAC;QACD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC;QACxC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;QACpC,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,MAAM,IAAI,KAAK,CAAC,yBAAyB,IAAI,EAAE,CAAC,CAAC;QACnD,CAAC;QACD,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;IACnB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,iBAAiB;IACxB,MAAM,GAAG,GAA2B,EAAE,CAAC;IACvC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QACvD,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9B,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QACnB,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,gBAAgB,CACvB,QAAwC;IAExC,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC7B,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;QAC3B,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC7B,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,cAAc,CAAC,QAAkB;IACxC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,QAAQ,CAAC;IACpC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QAC3C,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,QAAQ,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC;QAC9D,OAAO,GAAG,OAAO,IAAI,OAAO,CAAC,CAAC,CAAC,IAAI,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;IACtD,CAAC;IACD,OAAO,OAAO,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,QAAQ,IAAI,OAAO,EAAE,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,QAAQ,EAAE,CAAC;AAC9E,CAAC;AAED,SAAS,oBAAoB,CAAC,MAAe;IAC3C,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;QAClD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,SAAS,GAAG,MAEjB,CAAC;IACF,IACE,SAAS,CAAC,iBAAiB;QAC3B,OAAO,SAAS,CAAC,iBAAiB,KAAK,QAAQ;QAC/C,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,iBAAiB,CAAC,EAC3C,CAAC;QACD,OAAO,SAAS,CAAC,iBAAiB,CAAC;IACrC,CAAC;IACD,IACE,SAAS,CAAC,OAAO;QACjB,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC;QAChC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,IAAI,KAAK,MAAM;QACrC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,EACzB,CAAC;QACD,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YACrD,IACE,OAAO,MAAM,KAAK,QAAQ;gBAC1B,MAAM,KAAK,IAAI;gBACf,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EACtB,CAAC;gBACD,OAAO,MAA2B,CAAC;YACrC,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,WAAW,CAAC,MAAe;IAClC,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;QAClD,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,SAAS,GAAG,MAAoB,CAAC;IACvC,OAAO,SAAS,CAAC,OAAO,KAAK,IAAI,CAAC;AACpC,CAAC;AAED,SAAS,cAAc,CACrB,UAAoC,EACpC,GAAW;IAEX,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;IAC9B,OAAO,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;AAClD,CAAC;AAED,MAAM,OAAO,GAAG;IACd,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,EAAE;IACrC,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IACxB,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;IACzB,UAAU,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IAC9B,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IAC/B,UAAU,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;IAC/B,KAAK,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;IAC1B,YAAY,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IAChC,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;IACzB,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IACvB,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE;IACzB,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IACvB,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IAC1B,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IACvB,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE;CAC/B,CAAC;AAEX,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,SAAS,CAAC;IACxC,gBAAgB,EAAE,IAAI;IACtB,OAAO;CACR,CAAC,CAAC;AAEH,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;IAChB,UAAU,EAAE,CAAC;IACb,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AAED,SAAS,UAAU,CAAC,WAAqB;IACvC,MAAM,GAAG,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;IAC3B,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChD,UAAU,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,cAAc,CAAC,CAAC;IAClC,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,SAAS,GAAG,UAAU,CAAC,WAAW,CAAC,CAAC;AAE1C,MAAM,YAAY,GAAG,MAAM,CAAC,YAAY,CAAC,CAAC;AAC1C,MAAM,cAAc,GAClB,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAChE,IAAI,YAAY,KAAK,SAAS,IAAI,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,CAAC;IAC/D,MAAM,IAAI,KAAK,CAAC,+BAA+B,YAAY,EAAE,CAAC,CAAC;AACjE,CAAC;AAED,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;AACtC,MAAM,OAAO,GAAG,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAC1E,IAAI,UAAU,KAAK,SAAS,IAAI,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;IACtD,MAAM,IAAI,KAAK,CAAC,6BAA6B,UAAU,EAAE,CAAC,CAAC;AAC7D,CAAC;AAED,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC;AACxC,MAAM,QAAQ,GAAG,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AAC7E,IAAI,WAAW,KAAK,SAAS,IAAI,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC;IACxD,MAAM,IAAI,KAAK,CAAC,8BAA8B,WAAW,EAAE,CAAC,CAAC;AAC/D,CAAC;AAED,MAAM,UAAU,GAAG,CAAC,QAAkB,EAAQ,EAAE;IAC9C,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,cAAc,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;AACnE,CAAC,CAAC;AAEF,KAAK,UAAU,GAAG;IAChB,IAAI,SAAS,GAAqB,IAAI,CAAC;IACvC,MAAM,MAAM,GAAG,IAAI,MAAM,CACvB,EAAE,IAAI,EAAE,sBAAsB,EAAE,OAAO,EAAE,OAAO,EAAE,EAClD,EAAE,YAAY,EAAE,EAAE,EAAE,CACrB,CAAC;IAEF,IAAI,CAAC;QACH,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;YAChB,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YACtC,SAAS,GAAG,IAAI,6BAA6B,CAAC,QAAQ,CAAc,CAAC;QACvE,CAAC;aAAM,CAAC;YACN,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,IAAI,OAAO,CAAC,QAAQ,CAAC;YAC/C,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,SAAS,CAAC,CAAC;YAC/C,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC;YACzE,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,IAAI,QAAQ,CAAC;YAEnC,IAAI,CAAC;gBACH,MAAM,MAAM,CAAC,UAAU,CAAC,CAAC;YAC3B,CAAC;YAAC,MAAM,CAAC;gBACP,MAAM,IAAI,KAAK,CACb,6BAA6B,UAAU,gDAAgD,CACxF,CAAC;YACJ,CAAC;YAED,MAAM,GAAG,GAAG;gBACV,GAAG,iBAAiB,EAAE;gBACtB,GAAG,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC;aACjC,CAAC;YAEF,SAAS,GAAG,IAAI,oBAAoB,CAAC;gBACnC,OAAO;gBACP,IAAI,EAAE,CAAC,UAAU,EAAE,SAAS,CAAC;gBAC7B,GAAG;gBACH,GAAG;gBACH,MAAM,EAAE,SAAS;aAClB,CAAc,CAAC;QAClB,CAAC;QAED,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QAEhC,MAAM,aAAa,GAKf;YACF,GAAG,EAAE,SAAS;YACd,gBAAgB,EAAE,MAAM,CAAC,UAAU,CAAC,IAAI,KAAK;YAC7C,YAAY,EAAE,MAAM,CAAC,KAAK,IAAI,KAAK;SACpC,CAAC;QAEF,IAAI,OAAO,cAAc,KAAK,QAAQ,EAAE,CAAC;YACvC,aAAa,CAAC,cAAc,GAAG,cAAc,CAAC;QAChD,CAAC;QAED,MAAM,WAAW,GAA4C,EAAE,CAAC;QAChE,IAAI,OAAO,OAAO,KAAK,QAAQ,EAAE,CAAC;YAChC,WAAW,CAAC,GAAG,GAAG,OAAO,CAAC;QAC5B,CAAC;QACD,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;YACjC,WAAW,CAAC,YAAY,GAAG,QAAQ,CAAC;QACtC,CAAC;QAED,MAAM,cAAc,GAAG,MAAM,CAAC,IAAI;YAChC,CAAC,CAAC,EAAE,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE,WAAW,EAAE;YAC/C,CAAC,CAAC,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC;QAE/B,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI;YACxB,CAAC,CAAC,MAAM,cAAc,CAAC,MAAM,EAAE,aAAa,EAAE,cAAc,CAAC;YAC7D,CAAC,CAAC,MAAM,MAAM,CAAC,QAAQ,CACnB;gBACE,IAAI,EAAE,WAAW;gBACjB,SAAS,EAAE,aAAa;aACzB,EACD,SAAS,EACT,cAAc,CACf,CAAC;QAEN,IAAI,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC;YACxB,MAAM,YAAY,GAAG,oBAAoB,CAAC,MAAM,CAAC,IAAI;gBACnD,OAAO,EAAE,cAAc;aACxB,CAAC;YACF,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,YAAY,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;YACnE,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;YACrB,OAAO;QACT,CAAC;QAED,MAAM,UAAU,GAAG,oBAAoB,CAAC,MAAM,CAAC,CAAC;QAEhD,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;YAChB,MAAM,OAAO,GAAG,UAAU,IAAI,MAAM,CAAC;YACrC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;YAC9D,OAAO;QACT,CAAC;QAED,IAAI,QAAQ,GAAG,cAAc,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;QACtD,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;YAChB,MAAM,gBAAgB,GAAG,cAAc,CAAC,UAAU,EAAE,kBAAkB,CAAC,CAAC;YACxE,IAAI,gBAAgB,EAAE,CAAC;gBACrB,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC;oBACzC,GAAG,EAAE,gBAAgB;iBACtB,CAAC,CAAC;gBACH,MAAM,IAAI,GAAG,gBAAgB,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;gBACjD,IAAI,IAAI,EAAE,CAAC;oBACT,QAAQ,GAAG,IAAI,CAAC;gBAClB,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;QAC1D,CAAC;QAED,IAAI,MAAM,CAAC,GAAG,EAAE,CAAC;YACf,MAAM,SAAS,CAAC,MAAM,CAAC,GAAG,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;QAChD,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,QAAQ,IAAI,CAAC,CAAC;QACxC,CAAC;IACH,CAAC;YAAS,CAAC;QACT,IAAI,SAAS,EAAE,CAAC;YACd,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;QAC1B,CAAC;IACH,CAAC;AACH,CAAC;AAED,KAAK,UAAU,cAAc,CAC3B,MAAc,EACd,aAKC,EACD,cAMC;IAED,MAAM,MAAM,GAAG,MAAM,CAAC,YAAY,CAAC,KAAK,CAAC,cAAc,CACrD;QACE,IAAI,EAAE,WAAW;QACjB,SAAS,EAAE,aAAa;KACzB,EACD,SAAS,EACT,cAAc,CACf,CAAC;IAEF,IAAI,WAAW,GAAY,IAAI,CAAC;IAEhC,IAAI,KAAK,EAAE,MAAM,OAAO,IAAI,MAAM,EAAE,CAAC;QACnC,IAAI,OAAO,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;YACnC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,kBAAkB,OAAO,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC;YAChE,SAAS;QACX,CAAC;QACD,IAAI,OAAO,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;YAClC,MAAM,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC,aAAa;gBAC9C,CAAC,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,aAAa,EAAE;gBAClC,CAAC,CAAC,EAAE,CAAC;YACP,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,OAAO,CAAC,IAAI,CAAC,MAAM,GAAG,aAAa,IAAI,CAAC,CAAC;YACxE,SAAS;QACX,CAAC;QACD,IAAI,OAAO,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC9B,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC;YAC7B,SAAS;QACX,CAAC;QACD,IAAI,OAAO,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,IAAI,aAAa,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;IAED,IAAI,WAAW,KAAK,IAAI,EAAE,CAAC;QACzB,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACzD,CAAC;IAED,OAAO,WAAW,CAAC;AACrB,CAAC;AAED,IAAI,CAAC;IACH,MAAM,GAAG,EAAE,CAAC;AACd,CAAC;AAAC,OAAO,KAAK,EAAE,CAAC;IACf,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACvE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,OAAO,IAAI,CAAC,CAAC;IACrC,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;AACvB,CAAC"}
@@ -1,5 +0,0 @@
1
- export declare function getCharsetFromContentType(contentType: string | null): string | undefined;
2
- export declare function decodeBuffer(buffer: Uint8Array, encoding: string): string;
3
- export declare function resolveEncoding(declaredEncoding: string | undefined, sample: Uint8Array): string | undefined;
4
- export declare function isBinaryContent(buffer: Uint8Array, encoding?: string): boolean;
5
- //# sourceMappingURL=fetch-content.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"fetch-content.d.ts","sourceRoot":"","sources":["../src/fetch-content.ts"],"names":[],"mappings":"AAIA,wBAAgB,yBAAyB,CACvC,WAAW,EAAE,MAAM,GAAG,IAAI,GACzB,MAAM,GAAG,SAAS,CAWpB;AAaD,wBAAgB,YAAY,CAAC,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAEzE;AAiHD,wBAAgB,eAAe,CAC7B,gBAAgB,EAAE,MAAM,GAAG,SAAS,EACpC,MAAM,EAAE,UAAU,GACjB,MAAM,GAAG,SAAS,CAOpB;AAyCD,wBAAgB,eAAe,CAC7B,MAAM,EAAE,UAAU,EAClB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAMT"}
@@ -1,164 +0,0 @@
1
- import { Buffer } from 'node:buffer';
2
- const UTF8_ENCODING = 'utf-8';
3
- export function getCharsetFromContentType(contentType) {
4
- if (!contentType)
5
- return undefined;
6
- const match = /charset=([^;]+)/i.exec(contentType);
7
- const charsetGroup = match?.[1];
8
- if (!charsetGroup)
9
- return undefined;
10
- let charset = charsetGroup.trim();
11
- if (charset.startsWith('"') && charset.endsWith('"')) {
12
- charset = charset.slice(1, -1);
13
- }
14
- return charset.trim();
15
- }
16
- function createDecoder(encoding) {
17
- const fallback = () => new TextDecoder(UTF8_ENCODING);
18
- if (!encoding)
19
- return fallback();
20
- try {
21
- return new TextDecoder(encoding);
22
- }
23
- catch {
24
- return fallback();
25
- }
26
- }
27
- export function decodeBuffer(buffer, encoding) {
28
- return createDecoder(encoding).decode(buffer);
29
- }
30
- function normalizeEncodingLabel(encoding) {
31
- return encoding?.trim().toLowerCase() ?? '';
32
- }
33
- function isUnicodeWideEncoding(encoding) {
34
- const normalized = normalizeEncodingLabel(encoding);
35
- return (normalized.startsWith('utf-16') ||
36
- normalized.startsWith('utf-32') ||
37
- normalized === 'ucs-2' ||
38
- normalized === 'unicodefffe' ||
39
- normalized === 'unicodefeff');
40
- }
41
- const BOM_SIGNATURES = [
42
- // 4-byte BOMs must come first to avoid false matches with 2-byte prefixes
43
- { bytes: [0xff, 0xfe, 0x00, 0x00], encoding: 'utf-32le' },
44
- { bytes: [0x00, 0x00, 0xfe, 0xff], encoding: 'utf-32be' },
45
- { bytes: [0xef, 0xbb, 0xbf], encoding: 'utf-8' },
46
- { bytes: [0xff, 0xfe], encoding: 'utf-16le' },
47
- { bytes: [0xfe, 0xff], encoding: 'utf-16be' },
48
- ];
49
- function startsWithBytes(buffer, signature) {
50
- const sigLen = signature.length;
51
- if (buffer.length < sigLen)
52
- return false;
53
- for (let i = 0; i < sigLen; i += 1) {
54
- if (buffer[i] !== signature[i])
55
- return false;
56
- }
57
- return true;
58
- }
59
- function detectBomEncoding(buffer) {
60
- for (const { bytes, encoding } of BOM_SIGNATURES) {
61
- if (startsWithBytes(buffer, bytes))
62
- return encoding;
63
- }
64
- return undefined;
65
- }
66
- function readQuotedValue(input, startIndex) {
67
- const first = input[startIndex];
68
- if (!first)
69
- return '';
70
- const quoted = first === '"' || first === "'";
71
- if (quoted) {
72
- const end = input.indexOf(first, startIndex + 1);
73
- return end === -1 ? '' : input.slice(startIndex + 1, end).trim();
74
- }
75
- const tail = input.slice(startIndex);
76
- const stop = tail.search(/[\s/>]/);
77
- return (stop === -1 ? tail : tail.slice(0, stop)).trim();
78
- }
79
- function findTokenValue(original, lower, token, fromIndex = 0) {
80
- const tokenIndex = lower.indexOf(token, fromIndex);
81
- if (tokenIndex === -1)
82
- return undefined;
83
- const valueStart = tokenIndex + token.length;
84
- const value = readQuotedValue(original, valueStart);
85
- return value || undefined;
86
- }
87
- function extractHtmlCharset(headSnippet) {
88
- const lower = headSnippet.toLowerCase();
89
- const charset = findTokenValue(headSnippet, lower, 'charset=');
90
- return charset ? charset.toLowerCase() : undefined;
91
- }
92
- function extractXmlEncoding(headSnippet) {
93
- const lower = headSnippet.toLowerCase();
94
- const xmlStart = lower.indexOf('<?xml');
95
- if (xmlStart === -1)
96
- return undefined;
97
- const xmlEnd = lower.indexOf('?>', xmlStart);
98
- const declaration = xmlEnd === -1
99
- ? headSnippet.slice(xmlStart)
100
- : headSnippet.slice(xmlStart, xmlEnd + 2);
101
- const declarationLower = declaration.toLowerCase();
102
- const encoding = findTokenValue(declaration, declarationLower, 'encoding=');
103
- return encoding ? encoding.toLowerCase() : undefined;
104
- }
105
- function detectHtmlDeclaredEncoding(buffer) {
106
- const scanSize = Math.min(buffer.length, 8_192);
107
- if (scanSize === 0)
108
- return undefined;
109
- const headSnippet = Buffer.from(buffer.buffer, buffer.byteOffset, scanSize).toString('latin1');
110
- return extractHtmlCharset(headSnippet) ?? extractXmlEncoding(headSnippet);
111
- }
112
- export function resolveEncoding(declaredEncoding, sample) {
113
- const bomEncoding = detectBomEncoding(sample);
114
- if (bomEncoding)
115
- return bomEncoding;
116
- if (declaredEncoding)
117
- return declaredEncoding;
118
- return detectHtmlDeclaredEncoding(sample);
119
- }
120
- const BINARY_SIGNATURES = [
121
- [0x25, 0x50, 0x44, 0x46],
122
- [0x89, 0x50, 0x4e, 0x47],
123
- [0x47, 0x49, 0x46, 0x38],
124
- [0xff, 0xd8, 0xff],
125
- [0x52, 0x49, 0x46, 0x46],
126
- [0x42, 0x4d],
127
- [0x49, 0x49, 0x2a, 0x00],
128
- [0x4d, 0x4d, 0x00, 0x2a],
129
- [0x00, 0x00, 0x01, 0x00],
130
- [0x50, 0x4b, 0x03, 0x04],
131
- [0x1f, 0x8b],
132
- [0x42, 0x5a, 0x68],
133
- [0x52, 0x61, 0x72, 0x21],
134
- [0x37, 0x7a, 0xbc, 0xaf],
135
- [0x7f, 0x45, 0x4c, 0x46],
136
- [0x4d, 0x5a],
137
- [0xcf, 0xfa, 0xed, 0xfe],
138
- [0x00, 0x61, 0x73, 0x6d],
139
- [0x1a, 0x45, 0xdf, 0xa3],
140
- [0x66, 0x74, 0x79, 0x70],
141
- [0x46, 0x4c, 0x56],
142
- [0x49, 0x44, 0x33],
143
- [0xff, 0xfb],
144
- [0xff, 0xfa],
145
- [0x4f, 0x67, 0x67, 0x53],
146
- [0x66, 0x4c, 0x61, 0x43],
147
- [0x4d, 0x54, 0x68, 0x64],
148
- [0x77, 0x4f, 0x46, 0x46],
149
- [0x00, 0x01, 0x00, 0x00],
150
- [0x4f, 0x54, 0x54, 0x4f],
151
- [0x53, 0x51, 0x4c, 0x69],
152
- ];
153
- function hasNullByte(buffer, limit) {
154
- const checkLen = Math.min(buffer.length, limit);
155
- return buffer.subarray(0, checkLen).includes(0x00);
156
- }
157
- export function isBinaryContent(buffer, encoding) {
158
- for (const signature of BINARY_SIGNATURES) {
159
- if (startsWithBytes(buffer, signature))
160
- return true;
161
- }
162
- return !isUnicodeWideEncoding(encoding) && hasNullByte(buffer, 1000);
163
- }
164
- //# sourceMappingURL=fetch-content.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"fetch-content.js","sourceRoot":"","sources":["../src/fetch-content.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC,MAAM,aAAa,GAAG,OAAO,CAAC;AAE9B,MAAM,UAAU,yBAAyB,CACvC,WAA0B;IAE1B,IAAI,CAAC,WAAW;QAAE,OAAO,SAAS,CAAC;IACnC,MAAM,KAAK,GAAG,kBAAkB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACnD,MAAM,YAAY,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC;IAEhC,IAAI,CAAC,YAAY;QAAE,OAAO,SAAS,CAAC;IACpC,IAAI,OAAO,GAAG,YAAY,CAAC,IAAI,EAAE,CAAC;IAClC,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACrD,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACjC,CAAC;IACD,OAAO,OAAO,CAAC,IAAI,EAAE,CAAC;AACxB,CAAC;AAED,SAAS,aAAa,CAAC,QAA4B;IACjD,MAAM,QAAQ,GAAG,GAAgB,EAAE,CAAC,IAAI,WAAW,CAAC,aAAa,CAAC,CAAC;IACnE,IAAI,CAAC,QAAQ;QAAE,OAAO,QAAQ,EAAE,CAAC;IAEjC,IAAI,CAAC;QACH,OAAO,IAAI,WAAW,CAAC,QAAQ,CAAC,CAAC;IACnC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,QAAQ,EAAE,CAAC;IACpB,CAAC;AACH,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,MAAkB,EAAE,QAAgB;IAC/D,OAAO,aAAa,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;AAChD,CAAC;AAED,SAAS,sBAAsB,CAAC,QAA4B;IAC1D,OAAO,QAAQ,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC;AAC9C,CAAC;AAED,SAAS,qBAAqB,CAAC,QAA4B;IACzD,MAAM,UAAU,GAAG,sBAAsB,CAAC,QAAQ,CAAC,CAAC;IACpD,OAAO,CACL,UAAU,CAAC,UAAU,CAAC,QAAQ,CAAC;QAC/B,UAAU,CAAC,UAAU,CAAC,QAAQ,CAAC;QAC/B,UAAU,KAAK,OAAO;QACtB,UAAU,KAAK,aAAa;QAC5B,UAAU,KAAK,aAAa,CAC7B,CAAC;AACJ,CAAC;AAED,MAAM,cAAc,GAGd;IACJ,0EAA0E;IAC1E,EAAE,KAAK,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE,QAAQ,EAAE,UAAU,EAAE;IACzD,EAAE,KAAK,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE,QAAQ,EAAE,UAAU,EAAE;IACzD,EAAE,KAAK,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE;IAChD,EAAE,KAAK,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,QAAQ,EAAE,UAAU,EAAE;IAC7C,EAAE,KAAK,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,QAAQ,EAAE,UAAU,EAAE;CAC9C,CAAC;AAEF,SAAS,eAAe,CACtB,MAAkB,EAClB,SAA4B;IAE5B,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC;IAChC,IAAI,MAAM,CAAC,MAAM,GAAG,MAAM;QAAE,OAAO,KAAK,CAAC;IAEzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QACnC,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC;IAC/C,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,iBAAiB,CAAC,MAAkB;IAC3C,KAAK,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,IAAI,cAAc,EAAE,CAAC;QACjD,IAAI,eAAe,CAAC,MAAM,EAAE,KAAK,CAAC;YAAE,OAAO,QAAQ,CAAC;IACtD,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,eAAe,CAAC,KAAa,EAAE,UAAkB;IACxD,MAAM,KAAK,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC;IAChC,IAAI,CAAC,KAAK;QAAE,OAAO,EAAE,CAAC;IAEtB,MAAM,MAAM,GAAG,KAAK,KAAK,GAAG,IAAI,KAAK,KAAK,GAAG,CAAC;IAC9C,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,UAAU,GAAG,CAAC,CAAC,CAAC;QACjD,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,UAAU,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IACnE,CAAC;IAED,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IACrC,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IACnC,OAAO,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;AAC3D,CAAC;AAED,SAAS,cAAc,CACrB,QAAgB,EAChB,KAAa,EACb,KAAa,EACb,SAAS,GAAG,CAAC;IAEb,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IACnD,IAAI,UAAU,KAAK,CAAC,CAAC;QAAE,OAAO,SAAS,CAAC;IAExC,MAAM,UAAU,GAAG,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC;IAC7C,MAAM,KAAK,GAAG,eAAe,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IACpD,OAAO,KAAK,IAAI,SAAS,CAAC;AAC5B,CAAC;AAED,SAAS,kBAAkB,CAAC,WAAmB;IAC7C,MAAM,KAAK,GAAG,WAAW,CAAC,WAAW,EAAE,CAAC;IACxC,MAAM,OAAO,GAAG,cAAc,CAAC,WAAW,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC;IAC/D,OAAO,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;AACrD,CAAC;AAED,SAAS,kBAAkB,CAAC,WAAmB;IAC7C,MAAM,KAAK,GAAG,WAAW,CAAC,WAAW,EAAE,CAAC;IACxC,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACxC,IAAI,QAAQ,KAAK,CAAC,CAAC;QAAE,OAAO,SAAS,CAAC;IAEtC,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAC7C,MAAM,WAAW,GACf,MAAM,KAAK,CAAC,CAAC;QACX,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC,QAAQ,CAAC;QAC7B,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC;IAC9C,MAAM,gBAAgB,GAAG,WAAW,CAAC,WAAW,EAAE,CAAC;IAEnD,MAAM,QAAQ,GAAG,cAAc,CAAC,WAAW,EAAE,gBAAgB,EAAE,WAAW,CAAC,CAAC;IAC5E,OAAO,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;AACvD,CAAC;AAED,SAAS,0BAA0B,CAAC,MAAkB;IACpD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAChD,IAAI,QAAQ,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IAErC,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAC7B,MAAM,CAAC,MAAM,EACb,MAAM,CAAC,UAAU,EACjB,QAAQ,CACT,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAErB,OAAO,kBAAkB,CAAC,WAAW,CAAC,IAAI,kBAAkB,CAAC,WAAW,CAAC,CAAC;AAC5E,CAAC;AAED,MAAM,UAAU,eAAe,CAC7B,gBAAoC,EACpC,MAAkB;IAElB,MAAM,WAAW,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC;IAC9C,IAAI,WAAW;QAAE,OAAO,WAAW,CAAC;IAEpC,IAAI,gBAAgB;QAAE,OAAO,gBAAgB,CAAC;IAE9C,OAAO,0BAA0B,CAAC,MAAM,CAAC,CAAC;AAC5C,CAAC;AAED,MAAM,iBAAiB,GAAG;IACxB,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC;IACxB,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC;IACxB,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC;IACxB,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC;IAClB,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC;IACxB,CAAC,IAAI,EAAE,IAAI,CAAC;IACZ,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC;IACxB,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC;IACxB,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC;IACxB,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC;IACxB,CAAC,IAAI,EAAE,IAAI,CAAC;IACZ,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC;IAClB,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC;IACxB,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC;IACxB,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC;IACxB,CAAC,IAAI,EAAE,IAAI,CAAC;IACZ,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC;IACxB,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC;IACxB,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC;IACxB,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC;IACxB,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC;IAClB,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC;IAClB,CAAC,IAAI,EAAE,IAAI,CAAC;IACZ,CAAC,IAAI,EAAE,IAAI,CAAC;IACZ,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC;IACxB,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC;IACxB,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC;IACxB,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC;IACxB,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC;IACxB,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC;IACxB,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC;CAChB,CAAC;AAEX,SAAS,WAAW,CAAC,MAAkB,EAAE,KAAa;IACpD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAChD,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;AACrD,CAAC;AAED,MAAM,UAAU,eAAe,CAC7B,MAAkB,EAClB,QAAiB;IAEjB,KAAK,MAAM,SAAS,IAAI,iBAAiB,EAAE,CAAC;QAC1C,IAAI,eAAe,CAAC,MAAM,EAAE,SAAS,CAAC;YAAE,OAAO,IAAI,CAAC;IACtD,CAAC;IAED,OAAO,CAAC,qBAAqB,CAAC,QAAQ,CAAC,IAAI,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;AACvE,CAAC"}
@@ -1,5 +0,0 @@
1
- import { Readable } from 'node:stream';
2
- import type { ReadableStream as NodeReadableStream } from 'node:stream/web';
3
- export declare function toNodeReadableStream(stream: ReadableStream<Uint8Array>, url: string, stage: string): NodeReadableStream<Uint8Array>;
4
- export declare function toWebReadableStream(stream: Readable, url: string, stage: string): ReadableStream<Uint8Array>;
5
- //# sourceMappingURL=fetch-stream.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"fetch-stream.d.ts","sourceRoot":"","sources":["../src/fetch-stream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,KAAK,EAAE,cAAc,IAAI,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AAiC5E,wBAAgB,oBAAoB,CAClC,MAAM,EAAE,cAAc,CAAC,UAAU,CAAC,EAClC,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,GACZ,kBAAkB,CAAC,UAAU,CAAC,CAGhC;AAED,wBAAgB,mBAAmB,CACjC,MAAM,EAAE,QAAQ,EAChB,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,GACZ,cAAc,CAAC,UAAU,CAAC,CAI5B"}
@@ -1,29 +0,0 @@
1
- import { Readable } from 'node:stream';
2
- import { FetchError } from './errors.js';
3
- import { isObject } from './type-guards.js';
4
- function isReadableStreamLike(value) {
5
- if (!isObject(value))
6
- return false;
7
- return (typeof value['getReader'] === 'function' &&
8
- typeof value['cancel'] === 'function' &&
9
- typeof value['tee'] === 'function' &&
10
- typeof value['locked'] === 'boolean');
11
- }
12
- function assertReadableStreamLike(stream, url, stage) {
13
- if (isReadableStreamLike(stream))
14
- return;
15
- throw new FetchError('Invalid response stream', url, 500, {
16
- reason: 'invalid_stream',
17
- stage,
18
- });
19
- }
20
- export function toNodeReadableStream(stream, url, stage) {
21
- assertReadableStreamLike(stream, url, stage);
22
- return stream;
23
- }
24
- export function toWebReadableStream(stream, url, stage) {
25
- const converted = Readable.toWeb(stream);
26
- assertReadableStreamLike(converted, url, stage);
27
- return converted;
28
- }
29
- //# sourceMappingURL=fetch-stream.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"fetch-stream.js","sourceRoot":"","sources":["../src/fetch-stream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAGvC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAK5C,SAAS,oBAAoB,CAC3B,KAAc;IAEd,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAEnC,OAAO,CACL,OAAO,KAAK,CAAC,WAAW,CAAC,KAAK,UAAU;QACxC,OAAO,KAAK,CAAC,QAAQ,CAAC,KAAK,UAAU;QACrC,OAAO,KAAK,CAAC,KAAK,CAAC,KAAK,UAAU;QAClC,OAAO,KAAK,CAAC,QAAQ,CAAC,KAAK,SAAS,CACrC,CAAC;AACJ,CAAC;AAED,SAAS,wBAAwB,CAC/B,MAAe,EACf,GAAW,EACX,KAAa;IAEb,IAAI,oBAAoB,CAAC,MAAM,CAAC;QAAE,OAAO;IACzC,MAAM,IAAI,UAAU,CAAC,yBAAyB,EAAE,GAAG,EAAE,GAAG,EAAE;QACxD,MAAM,EAAE,gBAAgB;QACxB,KAAK;KACN,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,MAAkC,EAClC,GAAW,EACX,KAAa;IAEb,wBAAwB,CAAC,MAAM,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;IAC7C,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,mBAAmB,CACjC,MAAgB,EAChB,GAAW,EACX,KAAa;IAEb,MAAM,SAAS,GAAY,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAClD,wBAAwB,CAAC,SAAS,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;IAChD,OAAO,SAAS,CAAC;AACnB,CAAC"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"fetch.d.ts","sourceRoot":"","sources":["../src/fetch.ts"],"names":[],"mappings":"AAoCA,UAAU,YAAY;IACpB,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,UAAU,eAAe;IACvB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC;IAC9B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;CAC5B;AAo4BD,UAAU,qBAAqB;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAymCD,wBAAgB,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAE/C;AAED,wBAAgB,YAAY,CAAC,SAAS,EAAE,MAAM,GAAG;IAC/C,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;CAClB,CAEA;AAED,wBAAgB,uBAAuB,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,CAEjE;AAED,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,eAAe,CAE9D;AAED,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAExD;AAED,wBAAgB,mBAAmB,CACjC,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,MAAM,GACb,qBAAqB,CAEvB;AAED,wBAAgB,mBAAmB,CACjC,OAAO,EAAE,qBAAqB,EAC9B,QAAQ,EAAE,QAAQ,EAClB,WAAW,CAAC,EAAE,MAAM,GACnB,IAAI,CAEN;AAED,wBAAgB,gBAAgB,CAC9B,OAAO,EAAE,qBAAqB,EAC9B,KAAK,EAAE,OAAO,EACd,MAAM,CAAC,EAAE,MAAM,GACd,IAAI,CAEN;AAED,wBAAsB,kBAAkB,CACtC,GAAG,EAAE,MAAM,EACX,IAAI,EAAE,WAAW,EACjB,YAAY,EAAE,MAAM,GACnB,OAAO,CAAC;IAAE,QAAQ,EAAE,QAAQ,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,CAAC,CAE9C;AAED,wBAAsB,gBAAgB,CACpC,QAAQ,EAAE,QAAQ,EAClB,GAAG,EAAE,MAAM,EACX,QAAQ,EAAE,MAAM,EAChB,MAAM,CAAC,EAAE,WAAW,EACpB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAA;CAAE,CAAC,CAUzC;AAED,wBAAsB,kBAAkB,CACtC,aAAa,EAAE,MAAM,EACrB,OAAO,CAAC,EAAE,YAAY,GACrB,OAAO,CAAC,MAAM,CAAC,CAEjB;AAED,wBAAsB,wBAAwB,CAC5C,aAAa,EAAE,MAAM,EACrB,OAAO,CAAC,EAAE,YAAY,GACrB,OAAO,CAAC;IACT,MAAM,EAAE,UAAU,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,OAAO,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;CAClB,CAAC,CAED"}