mcp-scraper 0.18.0 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/README.md +2 -2
  2. package/dist/bin/api-server.cjs +635 -109
  3. package/dist/bin/api-server.cjs.map +1 -1
  4. package/dist/bin/api-server.js +3 -3
  5. package/dist/bin/mcp-scraper-cli.cjs +1 -1
  6. package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
  7. package/dist/bin/mcp-scraper-cli.js +1 -1
  8. package/dist/bin/mcp-scraper-install.cjs +2 -2
  9. package/dist/bin/mcp-scraper-install.cjs.map +1 -1
  10. package/dist/bin/mcp-scraper-install.js +2 -2
  11. package/dist/bin/mcp-stdio-server.cjs +2228 -1783
  12. package/dist/bin/mcp-stdio-server.cjs.map +1 -1
  13. package/dist/bin/mcp-stdio-server.js +5 -4
  14. package/dist/bin/mcp-stdio-server.js.map +1 -1
  15. package/dist/bin/paa-harvest.cjs.map +1 -1
  16. package/dist/bin/paa-harvest.js +3 -3
  17. package/dist/{chunk-GL4BW4CP.js → chunk-22VEGGTW.js} +21 -9
  18. package/dist/{chunk-GL4BW4CP.js.map → chunk-22VEGGTW.js.map} +1 -1
  19. package/dist/{chunk-RUGJE5EB.js → chunk-EJK25QOW.js} +2 -2
  20. package/dist/chunk-JWIE5NCR.js +284 -0
  21. package/dist/chunk-JWIE5NCR.js.map +1 -0
  22. package/dist/{chunk-FMEDPBIV.js → chunk-KE7KE2Q2.js} +482 -35
  23. package/dist/chunk-KE7KE2Q2.js.map +1 -0
  24. package/dist/chunk-PZB3TJWK.js +7 -0
  25. package/dist/chunk-PZB3TJWK.js.map +1 -0
  26. package/dist/{chunk-BX5RCOG5.js → chunk-Q44DN6T2.js} +2 -2
  27. package/dist/{chunk-BX5RCOG5.js.map → chunk-Q44DN6T2.js.map} +1 -1
  28. package/dist/chunk-UN7VMHZL.js +244 -0
  29. package/dist/chunk-UN7VMHZL.js.map +1 -0
  30. package/dist/{chunk-RMPPYKUV.js → chunk-XDFSLSSH.js} +7 -218
  31. package/dist/chunk-XDFSLSSH.js.map +1 -0
  32. package/dist/{chunk-HE2LQPJ2.js → chunk-ZRKFW5FB.js} +2 -2
  33. package/dist/{db-LIOTIWVN.js → db-YHZYG7D2.js} +2 -2
  34. package/dist/{extract-bundle-U4D5LW5W.js → extract-bundle-OSUPAHCE.js} +58 -5
  35. package/dist/extract-bundle-OSUPAHCE.js.map +1 -0
  36. package/dist/index.cjs.map +1 -1
  37. package/dist/index.js +5 -4
  38. package/dist/index.js.map +1 -1
  39. package/dist/{server-MQDCAR6I.js → server-VDWIPV7F.js} +100 -363
  40. package/dist/server-VDWIPV7F.js.map +1 -0
  41. package/dist/{site-extract-repository-NVSZH35Y.js → site-extract-repository-GWKGK46Z.js} +3 -3
  42. package/dist/{worker-JQTS437L.js → worker-O5PZTTPY.js} +8 -8
  43. package/docs/mcp-tool-manifest.generated.json +564 -30
  44. package/docs/specs/meta-ad-creative-media-resolution-spec.md +31 -0
  45. package/docs/specs/unified-credit-and-scheduled-execution-billing-spec.md +1007 -0
  46. package/package.json +1 -1
  47. package/dist/chunk-FMEDPBIV.js.map +0 -1
  48. package/dist/chunk-HPV4VOQX.js +0 -27
  49. package/dist/chunk-HPV4VOQX.js.map +0 -1
  50. package/dist/chunk-RLWCHLV3.js +0 -7
  51. package/dist/chunk-RLWCHLV3.js.map +0 -1
  52. package/dist/chunk-RMPPYKUV.js.map +0 -1
  53. package/dist/extract-bundle-U4D5LW5W.js.map +0 -1
  54. package/dist/server-MQDCAR6I.js.map +0 -1
  55. /package/dist/{chunk-RUGJE5EB.js.map → chunk-EJK25QOW.js.map} +0 -0
  56. /package/dist/{chunk-HE2LQPJ2.js.map → chunk-ZRKFW5FB.js.map} +0 -0
  57. /package/dist/{db-LIOTIWVN.js.map → db-YHZYG7D2.js.map} +0 -0
  58. /package/dist/{site-extract-repository-NVSZH35Y.js.map → site-extract-repository-GWKGK46Z.js.map} +0 -0
  59. /package/dist/{worker-JQTS437L.js.map → worker-O5PZTTPY.js.map} +0 -0
@@ -0,0 +1,284 @@
1
+ // src/lib/media-extractor.ts
2
+ import { createWriteStream, mkdirSync } from "fs";
3
+ import { homedir } from "os";
4
+ import { join, extname, basename } from "path";
5
+ import { pipeline } from "stream/promises";
6
+ import { Readable } from "stream";
7
+ var AD_PATTERNS = [
8
+ "doubleclick.net",
9
+ "googlesyndication.com",
10
+ "googletagmanager.com",
11
+ "google-analytics.com",
12
+ "googletagservices.com",
13
+ "adservice.google",
14
+ "googletag.",
15
+ "pagead2.googlesyndication",
16
+ "facebook.net/tr",
17
+ "connect.facebook.net",
18
+ "fbcdn.net/rsrc",
19
+ "analytics.twitter.com",
20
+ "static.ads-twitter.com",
21
+ "ads.twitter.com",
22
+ "t.co/i/adsct",
23
+ "pixel.advertising.com",
24
+ "hotjar.com",
25
+ "clarity.ms",
26
+ "quantserve.com",
27
+ "scorecardresearch.com",
28
+ "newrelic.com",
29
+ "nr-data.net",
30
+ "segment.io",
31
+ "segment.com",
32
+ "amplitude.com",
33
+ "mixpanel.com",
34
+ "heap.io",
35
+ "fullstory.com",
36
+ "moatads.com",
37
+ "criteo.com",
38
+ "adsrvr.org",
39
+ "rubiconproject.com",
40
+ "pubmatic.com",
41
+ "openx.net",
42
+ "appnexus.com",
43
+ "amazon-adsystem.com",
44
+ "media.net",
45
+ "yieldmo.com",
46
+ "triplelift.com",
47
+ "sharethrough.com",
48
+ "prebid.",
49
+ "smaato.net",
50
+ "indexworm.com",
51
+ "casalemedia.com",
52
+ "outbrain.com",
53
+ "taboola.com",
54
+ "revcontent.com",
55
+ "mgid.com",
56
+ "tawk.to",
57
+ "intercom.io",
58
+ "drift.com",
59
+ "hs-scripts.com",
60
+ "zopim.com",
61
+ "livechatinc.com",
62
+ "userlike.com",
63
+ "onetrust.com",
64
+ "cookielaw.org",
65
+ "cookieinformation.com",
66
+ "trustarc.com",
67
+ "/ads/",
68
+ "/ad/",
69
+ "/banner/",
70
+ "/banners/",
71
+ "/pixel/",
72
+ "/beacon/",
73
+ "/tracking/",
74
+ "/tracker/",
75
+ "/remarketing/",
76
+ "/conversion/",
77
+ "1x1.gif",
78
+ "spacer.gif",
79
+ "blank.gif",
80
+ "transparent.gif"
81
+ ];
82
+ var IMAGE_EXTS = /* @__PURE__ */ new Set([".jpg", ".jpeg", ".png", ".webp", ".gif", ".avif", ".svg", ".tiff"]);
83
+ var VIDEO_EXTS = /* @__PURE__ */ new Set([".mp4", ".webm", ".mov", ".avi", ".m4v", ".ogv", ".mkv"]);
84
+ var AUDIO_EXTS = /* @__PURE__ */ new Set([".mp3", ".wav", ".ogg", ".aac", ".m4a", ".flac", ".opus"]);
85
+ function isAdUrl(url) {
86
+ const lower = url.toLowerCase();
87
+ return AD_PATTERNS.some((p) => lower.includes(p));
88
+ }
89
+ function isDataUri(url) {
90
+ return url.startsWith("data:");
91
+ }
92
+ function typeFromUrl(url) {
93
+ try {
94
+ const ext = extname(new URL(url).pathname).toLowerCase();
95
+ if (IMAGE_EXTS.has(ext)) return "image";
96
+ if (VIDEO_EXTS.has(ext)) return "video";
97
+ if (AUDIO_EXTS.has(ext)) return "audio";
98
+ } catch {
99
+ }
100
+ return null;
101
+ }
102
+ function typeFromMime(mime) {
103
+ const lower = mime.toLowerCase();
104
+ if (lower.startsWith("image/")) return "image";
105
+ if (lower.startsWith("video/")) return "video";
106
+ if (lower.startsWith("audio/")) return "audio";
107
+ return null;
108
+ }
109
+ function resolveUrl(raw, base) {
110
+ if (!raw || isDataUri(raw)) return null;
111
+ try {
112
+ return new URL(raw, base).href;
113
+ } catch {
114
+ return null;
115
+ }
116
+ }
117
+ function safeFilename(url, index) {
118
+ try {
119
+ const u = new URL(url);
120
+ const base = basename(u.pathname).replace(/[^a-zA-Z0-9._-]/g, "_").slice(0, 80);
121
+ return base || `asset-${index}`;
122
+ } catch {
123
+ return `asset-${index}`;
124
+ }
125
+ }
126
+ function extractMediaUrls(html, baseUrl) {
127
+ const seen = /* @__PURE__ */ new Set();
128
+ const urls = [];
129
+ const add = (raw) => {
130
+ const resolved = resolveUrl(raw.trim(), baseUrl);
131
+ if (!resolved || seen.has(resolved) || isAdUrl(resolved)) return;
132
+ seen.add(resolved);
133
+ urls.push(resolved);
134
+ };
135
+ for (const m of html.matchAll(/<img\s[^>]*>/gi)) {
136
+ const tag = m[0];
137
+ for (const attr of ["src", "data-src", "data-lazy-src", "data-original"]) {
138
+ const v = (tag.match(new RegExp(`\\b${attr}\\s*=\\s*["']([^"']+)["']`, "i")) ?? [])[1];
139
+ if (v) add(v);
140
+ }
141
+ const srcset = (tag.match(/\bsrcset\s*=\s*["']([^"']+)["']/i) ?? [])[1];
142
+ if (srcset) {
143
+ for (const part of srcset.split(",")) {
144
+ const u = part.trim().split(/\s+/)[0];
145
+ if (u) add(u);
146
+ }
147
+ }
148
+ }
149
+ for (const m of html.matchAll(/<source\s[^>]*>/gi)) {
150
+ const tag = m[0];
151
+ const src = (tag.match(/\bsrc\s*=\s*["']([^"']+)["']/i) ?? [])[1];
152
+ if (src) add(src);
153
+ const srcset = (tag.match(/\bsrcset\s*=\s*["']([^"']+)["']/i) ?? [])[1];
154
+ if (srcset) {
155
+ for (const part of srcset.split(",")) {
156
+ const u = part.trim().split(/\s+/)[0];
157
+ if (u) add(u);
158
+ }
159
+ }
160
+ }
161
+ for (const m of html.matchAll(/<video\s[^>]*>/gi)) {
162
+ const tag = m[0];
163
+ for (const attr of ["src", "poster"]) {
164
+ const v = (tag.match(new RegExp(`\\b${attr}\\s*=\\s*["']([^"']+)["']`, "i")) ?? [])[1];
165
+ if (v) add(v);
166
+ }
167
+ }
168
+ for (const m of html.matchAll(/<audio\s[^>]*>/gi)) {
169
+ const tag = m[0];
170
+ const v = (tag.match(/\bsrc\s*=\s*["']([^"']+)["']/i) ?? [])[1];
171
+ if (v) add(v);
172
+ }
173
+ for (const m of html.matchAll(/<meta\s[^>]*>/gi)) {
174
+ const tag = m[0];
175
+ const prop = (tag.match(/\b(?:property|name)\s*=\s*["']([^"']+)["']/i) ?? [])[1]?.toLowerCase();
176
+ if (prop === "og:image" || prop === "twitter:image") {
177
+ const content = (tag.match(/\bcontent\s*=\s*["']([^"']+)["']/i) ?? [])[1];
178
+ if (content) add(content);
179
+ }
180
+ }
181
+ for (const m of html.matchAll(/background(?:-image)?\s*:\s*url\(\s*["']?([^"')]+)["']?\s*\)/gi)) {
182
+ add(m[1]);
183
+ }
184
+ return urls;
185
+ }
186
+ async function downloadAsset(url, destDir, filename) {
187
+ const res = await fetch(url, {
188
+ headers: { "User-Agent": "Mozilla/5.0 (compatible; ThorbitBot/1.0)" },
189
+ signal: AbortSignal.timeout(15e3)
190
+ });
191
+ if (!res.ok) throw new Error(`HTTP ${res.status}`);
192
+ if (!res.body) throw new Error("Empty response body");
193
+ const mimeType = res.headers.get("content-type")?.split(";")[0].trim() ?? null;
194
+ let dest = join(destDir, filename);
195
+ if (mimeType && !extname(filename)) {
196
+ const mimeExt = {
197
+ "image/jpeg": ".jpg",
198
+ "image/png": ".png",
199
+ "image/webp": ".webp",
200
+ "image/gif": ".gif",
201
+ "image/svg+xml": ".svg",
202
+ "image/avif": ".avif",
203
+ "video/mp4": ".mp4",
204
+ "video/webm": ".webm",
205
+ "audio/mpeg": ".mp3",
206
+ "audio/ogg": ".ogg",
207
+ "audio/wav": ".wav"
208
+ };
209
+ const ext = mimeExt[mimeType];
210
+ if (ext) dest = dest + ext;
211
+ }
212
+ const writer = createWriteStream(dest);
213
+ await pipeline(Readable.fromWeb(res.body), writer);
214
+ const { statSync } = await import("fs");
215
+ const sizeBytes = statSync(dest).size;
216
+ return { savedPath: dest, sizeBytes, mimeType };
217
+ }
218
+ async function harvestPageMedia(html, pageUrl, options = {}) {
219
+ const types = options.types ?? ["image", "video", "audio"];
220
+ const typesSet = new Set(types);
221
+ const rawUrls = extractMediaUrls(html, pageUrl);
222
+ const totalFound = rawUrls.length;
223
+ const filteredUrls = [];
224
+ const kept = [];
225
+ for (const url of rawUrls) {
226
+ const type = typeFromUrl(url);
227
+ if (!type || !typesSet.has(type)) {
228
+ filteredUrls.push(url);
229
+ continue;
230
+ }
231
+ kept.push({ url, type });
232
+ }
233
+ if (options.outputDir === null) {
234
+ return {
235
+ pageUrl,
236
+ outputDir: null,
237
+ assets: kept.map(({ url, type }, i) => ({
238
+ url,
239
+ type,
240
+ mimeType: null,
241
+ filename: safeFilename(url, i),
242
+ savedPath: null,
243
+ sizeBytes: null
244
+ })),
245
+ filteredCount: filteredUrls.length,
246
+ totalFound
247
+ };
248
+ }
249
+ const domain = (() => {
250
+ try {
251
+ return new URL(pageUrl).hostname.replace(/^www\./, "");
252
+ } catch {
253
+ return "unknown";
254
+ }
255
+ })();
256
+ const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-").slice(0, 19);
257
+ const outDir = options.outputDir ?? join(homedir(), "Downloads", "mcp-scraper", "media", `${stamp}-${domain}`);
258
+ mkdirSync(outDir, { recursive: true });
259
+ const assets = [];
260
+ await Promise.allSettled(
261
+ kept.map(async ({ url, type }, i) => {
262
+ const filename = safeFilename(url, i);
263
+ try {
264
+ const { savedPath, sizeBytes, mimeType } = await downloadAsset(url, outDir, filename);
265
+ const resolvedType = mimeType ? typeFromMime(mimeType) ?? type : type;
266
+ assets.push({ url, type: resolvedType, mimeType, filename: basename(savedPath), savedPath, sizeBytes });
267
+ } catch {
268
+ assets.push({ url, type, mimeType: null, filename, savedPath: null, sizeBytes: null });
269
+ }
270
+ })
271
+ );
272
+ assets.sort((a, b) => {
273
+ if (a.savedPath && !b.savedPath) return -1;
274
+ if (!a.savedPath && b.savedPath) return 1;
275
+ return a.url.localeCompare(b.url);
276
+ });
277
+ return { pageUrl, outputDir: outDir, assets, filteredCount: filteredUrls.length, totalFound };
278
+ }
279
+
280
+ export {
281
+ downloadAsset,
282
+ harvestPageMedia
283
+ };
284
+ //# sourceMappingURL=chunk-JWIE5NCR.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["../src/lib/media-extractor.ts"],"sourcesContent":["import { createWriteStream, mkdirSync } from 'node:fs'\nimport { homedir } from 'node:os'\nimport { join, extname, basename } from 'node:path'\nimport { pipeline } from 'node:stream/promises'\nimport { Readable } from 'node:stream'\n\nconst AD_PATTERNS: string[] = [\n 'doubleclick.net', 'googlesyndication.com', 'googletagmanager.com',\n 'google-analytics.com', 'googletagservices.com', 'adservice.google',\n 'googletag.', 'pagead2.googlesyndication',\n 'facebook.net/tr', 'connect.facebook.net', 'fbcdn.net/rsrc',\n 'analytics.twitter.com', 'static.ads-twitter.com', 'ads.twitter.com',\n 't.co/i/adsct', 'pixel.advertising.com',\n 'hotjar.com', 'clarity.ms', 'quantserve.com', 'scorecardresearch.com',\n 'newrelic.com', 'nr-data.net', 'segment.io', 'segment.com',\n 'amplitude.com', 'mixpanel.com', 'heap.io', 'fullstory.com',\n 'moatads.com', 'criteo.com', 'adsrvr.org', 'rubiconproject.com',\n 'pubmatic.com', 'openx.net', 'appnexus.com', 'amazon-adsystem.com',\n 'media.net', 'yieldmo.com', 'triplelift.com', 'sharethrough.com',\n 'prebid.', 'smaato.net', 'indexworm.com', 'casalemedia.com',\n 'outbrain.com', 'taboola.com', 'revcontent.com', 'mgid.com',\n 'tawk.to', 'intercom.io', 'drift.com', 'hs-scripts.com',\n 'zopim.com', 'livechatinc.com', 'userlike.com',\n 'onetrust.com', 'cookielaw.org', 'cookieinformation.com', 'trustarc.com',\n '/ads/', '/ad/', '/banner/', '/banners/', '/pixel/', '/beacon/',\n '/tracking/', '/tracker/', '/remarketing/', '/conversion/',\n '1x1.gif', 'spacer.gif', 'blank.gif', 'transparent.gif',\n]\n\nconst IMAGE_EXTS = new Set(['.jpg', '.jpeg', '.png', '.webp', '.gif', '.avif', '.svg', '.tiff'])\nconst VIDEO_EXTS = new Set(['.mp4', '.webm', '.mov', '.avi', '.m4v', '.ogv', '.mkv'])\nconst AUDIO_EXTS = new Set(['.mp3', '.wav', '.ogg', '.aac', '.m4a', '.flac', '.opus'])\n\nexport type MediaType = 'image' | 'video' | 'audio'\n\nexport interface MediaAsset {\n url: string\n type: MediaType\n mimeType: string | null\n filename: string\n savedPath: string | null\n sizeBytes: number | null\n}\n\nexport interface MediaManifest {\n pageUrl: string\n outputDir: string | null\n assets: MediaAsset[]\n filteredCount: number\n totalFound: number\n}\n\nexport interface MediaExtractOptions {\n types?: MediaType[]\n outputDir?: string | null\n}\n\nfunction isAdUrl(url: string): boolean {\n const lower = url.toLowerCase()\n return AD_PATTERNS.some(p => lower.includes(p))\n}\n\nfunction isDataUri(url: string): boolean {\n return url.startsWith('data:')\n}\n\nfunction typeFromUrl(url: string): MediaType | null {\n try {\n const ext = extname(new URL(url).pathname).toLowerCase()\n if (IMAGE_EXTS.has(ext)) return 'image'\n if (VIDEO_EXTS.has(ext)) return 'video'\n if (AUDIO_EXTS.has(ext)) return 'audio'\n } catch { /* non-parseable URL */ }\n return null\n}\n\nfunction typeFromMime(mime: string): MediaType | null {\n const lower = mime.toLowerCase()\n if (lower.startsWith('image/')) return 'image'\n if (lower.startsWith('video/')) return 'video'\n if (lower.startsWith('audio/')) return 'audio'\n return null\n}\n\nfunction resolveUrl(raw: string, base: string): string | null {\n if (!raw || isDataUri(raw)) return null\n try { return new URL(raw, base).href } catch { return null }\n}\n\nfunction safeFilename(url: string, index: number): string {\n try {\n const u = new URL(url)\n const base = basename(u.pathname).replace(/[^a-zA-Z0-9._-]/g, '_').slice(0, 80)\n return base || `asset-${index}`\n } catch {\n return `asset-${index}`\n }\n}\n\nexport function extractMediaUrls(html: string, baseUrl: string): string[] {\n const seen = new Set<string>()\n const urls: string[] = []\n\n const add = (raw: string) => {\n const resolved = resolveUrl(raw.trim(), baseUrl)\n if (!resolved || seen.has(resolved) || isAdUrl(resolved)) return\n seen.add(resolved)\n urls.push(resolved)\n }\n\n for (const m of html.matchAll(/<img\\s[^>]*>/gi)) {\n const tag = m[0]\n for (const attr of ['src', 'data-src', 'data-lazy-src', 'data-original']) {\n const v = (tag.match(new RegExp(`\\\\b${attr}\\\\s*=\\\\s*[\"']([^\"']+)[\"']`, 'i')) ?? [])[1]\n if (v) add(v)\n }\n const srcset = (tag.match(/\\bsrcset\\s*=\\s*[\"']([^\"']+)[\"']/i) ?? [])[1]\n if (srcset) {\n for (const part of srcset.split(',')) {\n const u = part.trim().split(/\\s+/)[0]\n if (u) add(u)\n }\n }\n }\n\n for (const m of html.matchAll(/<source\\s[^>]*>/gi)) {\n const tag = m[0]\n const src = (tag.match(/\\bsrc\\s*=\\s*[\"']([^\"']+)[\"']/i) ?? [])[1]\n if (src) add(src)\n const srcset = (tag.match(/\\bsrcset\\s*=\\s*[\"']([^\"']+)[\"']/i) ?? [])[1]\n if (srcset) {\n for (const part of srcset.split(',')) {\n const u = part.trim().split(/\\s+/)[0]\n if (u) add(u)\n }\n }\n }\n\n for (const m of html.matchAll(/<video\\s[^>]*>/gi)) {\n const tag = m[0]\n for (const attr of ['src', 'poster']) {\n const v = (tag.match(new RegExp(`\\\\b${attr}\\\\s*=\\\\s*[\"']([^\"']+)[\"']`, 'i')) ?? [])[1]\n if (v) add(v)\n }\n }\n\n for (const m of html.matchAll(/<audio\\s[^>]*>/gi)) {\n const tag = m[0]\n const v = (tag.match(/\\bsrc\\s*=\\s*[\"']([^\"']+)[\"']/i) ?? [])[1]\n if (v) add(v)\n }\n\n for (const m of html.matchAll(/<meta\\s[^>]*>/gi)) {\n const tag = m[0]\n const prop = (tag.match(/\\b(?:property|name)\\s*=\\s*[\"']([^\"']+)[\"']/i) ?? [])[1]?.toLowerCase()\n if (prop === 'og:image' || prop === 'twitter:image') {\n const content = (tag.match(/\\bcontent\\s*=\\s*[\"']([^\"']+)[\"']/i) ?? [])[1]\n if (content) add(content)\n }\n }\n\n for (const m of html.matchAll(/background(?:-image)?\\s*:\\s*url\\(\\s*[\"']?([^\"')]+)[\"']?\\s*\\)/gi)) {\n add(m[1])\n }\n\n return urls\n}\n\nexport async function downloadAsset(\n url: string,\n destDir: string,\n filename: string,\n): Promise<{ savedPath: string; sizeBytes: number; mimeType: string | null }> {\n const res = await fetch(url, {\n headers: { 'User-Agent': 'Mozilla/5.0 (compatible; ThorbitBot/1.0)' },\n signal: AbortSignal.timeout(15_000),\n })\n if (!res.ok) throw new Error(`HTTP ${res.status}`)\n if (!res.body) throw new Error('Empty response body')\n\n const mimeType = res.headers.get('content-type')?.split(';')[0].trim() ?? null\n\n let dest = join(destDir, filename)\n if (mimeType && !extname(filename)) {\n const mimeExt: Record<string, string> = {\n 'image/jpeg': '.jpg', 'image/png': '.png', 'image/webp': '.webp',\n 'image/gif': '.gif', 'image/svg+xml': '.svg', 'image/avif': '.avif',\n 'video/mp4': '.mp4', 'video/webm': '.webm',\n 'audio/mpeg': '.mp3', 'audio/ogg': '.ogg', 'audio/wav': '.wav',\n }\n const ext = mimeExt[mimeType]\n if (ext) dest = dest + ext\n }\n\n const writer = createWriteStream(dest)\n await pipeline(Readable.fromWeb(res.body as import('stream/web').ReadableStream), writer)\n\n const { statSync } = await import('node:fs')\n const sizeBytes = statSync(dest).size\n\n return { savedPath: dest, sizeBytes, mimeType }\n}\n\nexport async function harvestPageMedia(\n html: string,\n pageUrl: string,\n options: MediaExtractOptions = {},\n): Promise<MediaManifest> {\n const types = options.types ?? ['image', 'video', 'audio']\n const typesSet = new Set(types)\n const rawUrls = extractMediaUrls(html, pageUrl)\n const totalFound = rawUrls.length\n\n const filteredUrls: string[] = []\n const kept: Array<{ url: string; type: MediaType }> = []\n\n for (const url of rawUrls) {\n const type = typeFromUrl(url)\n if (!type || !typesSet.has(type)) { filteredUrls.push(url); continue }\n kept.push({ url, type })\n }\n\n if (options.outputDir === null) {\n return {\n pageUrl,\n outputDir: null,\n assets: kept.map(({ url, type }, i) => ({\n url, type, mimeType: null, filename: safeFilename(url, i), savedPath: null, sizeBytes: null,\n })),\n filteredCount: filteredUrls.length,\n totalFound,\n }\n }\n\n const domain = (() => { try { return new URL(pageUrl).hostname.replace(/^www\\./, '') } catch { return 'unknown' } })()\n const stamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19)\n const outDir = options.outputDir ?? join(homedir(), 'Downloads', 'mcp-scraper', 'media', `${stamp}-${domain}`)\n mkdirSync(outDir, { recursive: true })\n\n const assets: MediaAsset[] = []\n\n await Promise.allSettled(\n kept.map(async ({ url, type }, i) => {\n const filename = safeFilename(url, i)\n try {\n const { savedPath, sizeBytes, mimeType } = await downloadAsset(url, outDir, filename)\n const resolvedType = mimeType ? (typeFromMime(mimeType) ?? type) : type\n assets.push({ url, type: resolvedType, mimeType, filename: basename(savedPath), savedPath, sizeBytes })\n } catch {\n assets.push({ url, type, mimeType: null, filename, savedPath: null, sizeBytes: null })\n }\n })\n )\n\n assets.sort((a, b) => {\n if (a.savedPath && !b.savedPath) return -1\n if (!a.savedPath && b.savedPath) return 1\n return a.url.localeCompare(b.url)\n })\n\n return { pageUrl, outputDir: outDir, assets, filteredCount: filteredUrls.length, totalFound }\n}\n"],"mappings":";AAAA,SAAS,mBAAmB,iBAAiB;AAC7C,SAAS,eAAe;AACxB,SAAS,MAAM,SAAS,gBAAgB;AACxC,SAAS,gBAAgB;AACzB,SAAS,gBAAgB;AAEzB,IAAM,cAAwB;AAAA,EAC5B;AAAA,EAAmB;AAAA,EAAyB;AAAA,EAC5C;AAAA,EAAwB;AAAA,EAAyB;AAAA,EACjD;AAAA,EAAc;AAAA,EACd;AAAA,EAAmB;AAAA,EAAwB;AAAA,EAC3C;AAAA,EAAyB;AAAA,EAA0B;AAAA,EACnD;AAAA,EAAgB;AAAA,EAChB;AAAA,EAAc;AAAA,EAAc;AAAA,EAAkB;AAAA,EAC9C;AAAA,EAAgB;AAAA,EAAe;AAAA,EAAc;AAAA,EAC7C;AAAA,EAAiB;AAAA,EAAgB;AAAA,EAAW;AAAA,EAC5C;AAAA,EAAe;AAAA,EAAc;AAAA,EAAc;AAAA,EAC3C;AAAA,EAAgB;AAAA,EAAa;AAAA,EAAgB;AAAA,EAC7C;AAAA,EAAa;AAAA,EAAe;AAAA,EAAkB;AAAA,EAC9C;AAAA,EAAW;AAAA,EAAc;AAAA,EAAiB;AAAA,EAC1C;AAAA,EAAgB;AAAA,EAAe;AAAA,EAAkB;AAAA,EACjD;AAAA,EAAW;AAAA,EAAe;AAAA,EAAa;AAAA,EACvC;AAAA,EAAa;AAAA,EAAmB;AAAA,EAChC;AAAA,EAAgB;AAAA,EAAiB;AAAA,EAAyB;AAAA,EAC1D;AAAA,EAAS;AAAA,EAAQ;AAAA,EAAY;AAAA,EAAa;AAAA,EAAW;AAAA,EACrD;AAAA,EAAc;AAAA,EAAa;AAAA,EAAiB;AAAA,EAC5C;AAAA,EAAW;AAAA,EAAc;AAAA,EAAa;AACxC;AAEA,IAAM,aAAc,oBAAI,IAAI,CAAC,QAAQ,SAAS,QAAQ,SAAS,QAAQ,SAAS,QAAQ,OAAO,CAAC;AAChG,IAAM,aAAc,oBAAI,IAAI,CAAC,QAAQ,SAAS,QAAQ,QAAQ,QAAQ,QAAQ,MAAM,CAAC;AACrF,IAAM,aAAc,oBAAI,IAAI,CAAC,QAAQ,QAAQ,QAAQ,QAAQ,QAAQ,SAAS,OAAO,CAAC;AA0BtF,SAAS,QAAQ,KAAsB;AACrC,QAAM,QAAQ,IAAI,YAAY;AAC9B,SAAO,YAAY,KAAK,OAAK,MAAM,SAAS,CAAC,CAAC;AAChD;AAEA,SAAS,UAAU,KAAsB;AACvC,SAAO,IAAI,WAAW,OAAO;AAC/B;AAEA,SAAS,YAAY,KAA+B;AAClD,MAAI;AACF,UAAM,MAAM,QAAQ,IAAI,IAAI,GAAG,EAAE,QAAQ,EAAE,YAAY;AACvD,QAAI,WAAW,IAAI,GAAG,EAAG,QAAO;AAChC,QAAI,WAAW,IAAI,GAAG,EAAG,QAAO;AAChC,QAAI,WAAW,IAAI,GAAG,EAAG,QAAO;AAAA,EAClC,QAAQ;AAAA,EAA0B;AAClC,SAAO;AACT;AAEA,SAAS,aAAa,MAAgC;AACpD,QAAM,QAAQ,KAAK,YAAY;AAC/B,MAAI,MAAM,WAAW,QAAQ,EAAG,QAAO;AACvC,MAAI,MAAM,WAAW,QAAQ,EAAG,QAAO;AACvC,MAAI,MAAM,WAAW,QAAQ,EAAG,QAAO;AACvC,SAAO;AACT;AAEA,SAAS,WAAW,KAAa,MAA6B;AAC5D,MAAI,CAAC,OAAO,UAAU,GAAG,EAAG,QAAO;AACnC,MAAI;AAAE,WAAO,IAAI,IAAI,KAAK,IAAI,EAAE;AAAA,EAAK,QAAQ;AAAE,WAAO;AAAA,EAAK;AAC7D;AAEA,SAAS,aAAa,KAAa,OAAuB;AACxD,MAAI;AACF,UAAM,IAAI,IAAI,IAAI,GAAG;AACrB,UAAM,OAAO,SAAS,EAAE,QAAQ,EAAE,QAAQ,oBAAoB,GAAG,EAAE,MAAM,GAAG,EAAE;AAC9E,WAAO,QAAQ,SAAS,KAAK;AAAA,EAC/B,QAAQ;AACN,WAAO,SAAS,KAAK;AAAA,EACvB;AACF;AAEO,SAAS,iBAAiB,MAAc,SAA2B;AACxE,QAAM,OAAO,oBAAI,IAAY;AAC7B,QAAM,OAAiB,CAAC;AAExB,QAAM,MAAM,CAAC,QAAgB;AAC3B,UAAM,WAAW,WAAW,IAAI,KAAK,GAAG,OAAO;AAC/C,QAAI,CAAC,YAAY,KAAK,IAAI,QAAQ,KAAK,QAAQ,QAAQ,EAAG;AAC1D,SAAK,IAAI,QAAQ;AACjB,SAAK,KAAK,QAAQ;AAAA,EACpB;AAEA,aAAW,KAAK,KAAK,SAAS,gBAAgB,GAAG;AAC/C,UAAM,MAAM,EAAE,CAAC;AACf,eAAW,QAAQ,CAAC,OAAO,YAAY,iBAAiB,eAAe,GAAG;AACxE,YAAM,KAAK,IAAI,MAAM,IAAI,OAAO,MAAM,IAAI,6BAA6B,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC;AACrF,UAAI,EAAG,KAAI,CAAC;AAAA,IACd;AACA,UAAM,UAAU,IAAI,MAAM,kCAAkC,KAAK,CAAC,GAAG,CAAC;AACtE,QAAI,QAAQ;AACV,iBAAW,QAAQ,OAAO,MAAM,GAAG,GAAG;AACpC,cAAM,IAAI,KAAK,KAAK,EAAE,MAAM,KAAK,EAAE,CAAC;AACpC,YAAI,EAAG,KAAI,CAAC;AAAA,MACd;AAAA,IACF;AAAA,EACF;AAEA,aAAW,KAAK,KAAK,SAAS,mBAAmB,GAAG;AAClD,UAAM,MAAM,EAAE,CAAC;AACf,UAAM,OAAO,IAAI,MAAM,+BAA+B,KAAK,CAAC,GAAG,CAAC;AAChE,QAAI,IAAK,KAAI,GAAG;AAChB,UAAM,UAAU,IAAI,MAAM,kCAAkC,KAAK,CAAC,GAAG,CAAC;AACtE,QAAI,QAAQ;AACV,iBAAW,QAAQ,OAAO,MAAM,GAAG,GAAG;AACpC,cAAM,IAAI,KAAK,KAAK,EAAE,MAAM,KAAK,EAAE,CAAC;AACpC,YAAI,EAAG,KAAI,CAAC;AAAA,MACd;AAAA,IACF;AAAA,EACF;AAEA,aAAW,KAAK,KAAK,SAAS,kBAAkB,GAAG;AACjD,UAAM,MAAM,EAAE,CAAC;AACf,eAAW,QAAQ,CAAC,OAAO,QAAQ,GAAG;AACpC,YAAM,KAAK,IAAI,MAAM,IAAI,OAAO,MAAM,IAAI,6BAA6B,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC;AACrF,UAAI,EAAG,KAAI,CAAC;AAAA,IACd;AAAA,EACF;AAEA,aAAW,KAAK,KAAK,SAAS,kBAAkB,GAAG;AACjD,UAAM,MAAM,EAAE,CAAC;AACf,UAAM,KAAK,IAAI,MAAM,+BAA+B,KAAK,CAAC,GAAG,CAAC;AAC9D,QAAI,EAAG,KAAI,CAAC;AAAA,EACd;AAEA,aAAW,KAAK,KAAK,SAAS,iBAAiB,GAAG;AAChD,UAAM,MAAM,EAAE,CAAC;AACf,UAAM,QAAQ,IAAI,MAAM,6CAA6C,KAAK,CAAC,GAAG,CAAC,GAAG,YAAY;AAC9F,QAAI,SAAS,cAAc,SAAS,iBAAiB;AACnD,YAAM,WAAW,IAAI,MAAM,mCAAmC,KAAK,CAAC,GAAG,CAAC;AACxE,UAAI,QAAS,KAAI,OAAO;AAAA,IAC1B;AAAA,EACF;AAEA,aAAW,KAAK,KAAK,SAAS,gEAAgE,GAAG;AAC/F,QAAI,EAAE,CAAC,CAAC;AAAA,EACV;AAEA,SAAO;AACT;AAEA,eAAsB,cACpB,KACA,SACA,UAC4E;AAC5E,QAAM,MAAM,MAAM,MAAM,KAAK;AAAA,IAC3B,SAAS,EAAE,cAAc,2CAA2C;AAAA,IACpE,QAAS,YAAY,QAAQ,IAAM;AAAA,EACrC,CAAC;AACD,MAAI,CAAC,IAAI,GAAI,OAAM,IAAI,MAAM,QAAQ,IAAI,MAAM,EAAE;AACjD,MAAI,CAAC,IAAI,KAAM,OAAM,IAAI,MAAM,qBAAqB;AAEpD,QAAM,WAAW,IAAI,QAAQ,IAAI,cAAc,GAAG,MAAM,GAAG,EAAE,CAAC,EAAE,KAAK,KAAK;AAE1E,MAAI,OAAO,KAAK,SAAS,QAAQ;AACjC,MAAI,YAAY,CAAC,QAAQ,QAAQ,GAAG;AAClC,UAAM,UAAkC;AAAA,MACtC,cAAc;AAAA,MAAQ,aAAa;AAAA,MAAQ,cAAc;AAAA,MACzD,aAAa;AAAA,MAAQ,iBAAiB;AAAA,MAAQ,cAAc;AAAA,MAC5D,aAAa;AAAA,MAAQ,cAAc;AAAA,MACnC,cAAc;AAAA,MAAQ,aAAa;AAAA,MAAQ,aAAa;AAAA,IAC1D;AACA,UAAM,MAAM,QAAQ,QAAQ;AAC5B,QAAI,IAAK,QAAO,OAAO;AAAA,EACzB;AAEA,QAAM,SAAS,kBAAkB,IAAI;AACrC,QAAM,SAAS,SAAS,QAAQ,IAAI,IAA2C,GAAG,MAAM;AAExF,QAAM,EAAE,SAAS,IAAI,MAAM,OAAO,IAAS;AAC3C,QAAM,YAAY,SAAS,IAAI,EAAE;AAEjC,SAAO,EAAE,WAAW,MAAM,WAAW,SAAS;AAChD;AAEA,eAAsB,iBACpB,MACA,SACA,UAA+B,CAAC,GACR;AACxB,QAAM,QAAW,QAAQ,SAAS,CAAC,SAAS,SAAS,OAAO;AAC5D,QAAM,WAAW,IAAI,IAAI,KAAK;AAC9B,QAAM,UAAW,iBAAiB,MAAM,OAAO;AAC/C,QAAM,aAAa,QAAQ;AAE3B,QAAM,eAAyB,CAAC;AAChC,QAAM,OAAgD,CAAC;AAEvD,aAAW,OAAO,SAAS;AACzB,UAAM,OAAO,YAAY,GAAG;AAC5B,QAAI,CAAC,QAAQ,CAAC,SAAS,IAAI,IAAI,GAAG;AAAE,mBAAa,KAAK,GAAG;AAAG;AAAA,IAAS;AACrE,SAAK,KAAK,EAAE,KAAK,KAAK,CAAC;AAAA,EACzB;AAEA,MAAI,QAAQ,cAAc,MAAM;AAC9B,WAAO;AAAA,MACL;AAAA,MACA,WAAW;AAAA,MACX,QAAQ,KAAK,IAAI,CAAC,EAAE,KAAK,KAAK,GAAG,OAAO;AAAA,QACtC;AAAA,QAAK;AAAA,QAAM,UAAU;AAAA,QAAM,UAAU,aAAa,KAAK,CAAC;AAAA,QAAG,WAAW;AAAA,QAAM,WAAW;AAAA,MACzF,EAAE;AAAA,MACF,eAAe,aAAa;AAAA,MAC5B;AAAA,IACF;AAAA,EACF;AAEA,QAAM,UAAW,MAAM;AAAE,QAAI;AAAE,aAAO,IAAI,IAAI,OAAO,EAAE,SAAS,QAAQ,UAAU,EAAE;AAAA,IAAE,QAAQ;AAAE,aAAO;AAAA,IAAU;AAAA,EAAE,GAAG;AACtH,QAAM,SAAU,oBAAI,KAAK,GAAE,YAAY,EAAE,QAAQ,SAAS,GAAG,EAAE,MAAM,GAAG,EAAE;AAC1E,QAAM,SAAU,QAAQ,aAAa,KAAK,QAAQ,GAAG,aAAa,eAAe,SAAS,GAAG,KAAK,IAAI,MAAM,EAAE;AAC9G,YAAU,QAAQ,EAAE,WAAW,KAAK,CAAC;AAErC,QAAM,SAAuB,CAAC;AAE9B,QAAM,QAAQ;AAAA,IACZ,KAAK,IAAI,OAAO,EAAE,KAAK,KAAK,GAAG,MAAM;AACnC,YAAM,WAAW,aAAa,KAAK,CAAC;AACpC,UAAI;AACF,cAAM,EAAE,WAAW,WAAW,SAAS,IAAI,MAAM,cAAc,KAAK,QAAQ,QAAQ;AACpF,cAAM,eAAe,WAAY,aAAa,QAAQ,KAAK,OAAQ;AACnE,eAAO,KAAK,EAAE,KAAK,MAAM,cAAc,UAAU,UAAU,SAAS,SAAS,GAAG,WAAW,UAAU,CAAC;AAAA,MACxG,QAAQ;AACN,eAAO,KAAK,EAAE,KAAK,MAAM,UAAU,MAAM,UAAU,WAAW,MAAM,WAAW,KAAK,CAAC;AAAA,MACvF;AAAA,IACF,CAAC;AAAA,EACH;AAEA,SAAO,KAAK,CAAC,GAAG,MAAM;AACpB,QAAI,EAAE,aAAa,CAAC,EAAE,UAAW,QAAO;AACxC,QAAI,CAAC,EAAE,aAAa,EAAE,UAAW,QAAO;AACxC,WAAO,EAAE,IAAI,cAAc,EAAE,GAAG;AAAA,EAClC,CAAC;AAED,SAAO,EAAE,SAAS,WAAW,QAAQ,QAAQ,eAAe,aAAa,QAAQ,WAAW;AAC9F;","names":[]}