mcp-scraper 0.18.0 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/bin/api-server.cjs +635 -109
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +3 -3
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +2 -2
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +2 -2
- package/dist/bin/mcp-stdio-server.cjs +2228 -1783
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +5 -4
- package/dist/bin/mcp-stdio-server.js.map +1 -1
- package/dist/bin/paa-harvest.cjs.map +1 -1
- package/dist/bin/paa-harvest.js +3 -3
- package/dist/{chunk-GL4BW4CP.js → chunk-22VEGGTW.js} +21 -9
- package/dist/{chunk-GL4BW4CP.js.map → chunk-22VEGGTW.js.map} +1 -1
- package/dist/{chunk-RUGJE5EB.js → chunk-EJK25QOW.js} +2 -2
- package/dist/chunk-JWIE5NCR.js +284 -0
- package/dist/chunk-JWIE5NCR.js.map +1 -0
- package/dist/{chunk-FMEDPBIV.js → chunk-KE7KE2Q2.js} +482 -35
- package/dist/chunk-KE7KE2Q2.js.map +1 -0
- package/dist/chunk-PZB3TJWK.js +7 -0
- package/dist/chunk-PZB3TJWK.js.map +1 -0
- package/dist/{chunk-BX5RCOG5.js → chunk-Q44DN6T2.js} +2 -2
- package/dist/{chunk-BX5RCOG5.js.map → chunk-Q44DN6T2.js.map} +1 -1
- package/dist/chunk-UN7VMHZL.js +244 -0
- package/dist/chunk-UN7VMHZL.js.map +1 -0
- package/dist/{chunk-RMPPYKUV.js → chunk-XDFSLSSH.js} +7 -218
- package/dist/chunk-XDFSLSSH.js.map +1 -0
- package/dist/{chunk-HE2LQPJ2.js → chunk-ZRKFW5FB.js} +2 -2
- package/dist/{db-LIOTIWVN.js → db-YHZYG7D2.js} +2 -2
- package/dist/{extract-bundle-U4D5LW5W.js → extract-bundle-OSUPAHCE.js} +58 -5
- package/dist/extract-bundle-OSUPAHCE.js.map +1 -0
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +5 -4
- package/dist/index.js.map +1 -1
- package/dist/{server-MQDCAR6I.js → server-VDWIPV7F.js} +100 -363
- package/dist/server-VDWIPV7F.js.map +1 -0
- package/dist/{site-extract-repository-NVSZH35Y.js → site-extract-repository-GWKGK46Z.js} +3 -3
- package/dist/{worker-JQTS437L.js → worker-O5PZTTPY.js} +8 -8
- package/docs/mcp-tool-manifest.generated.json +564 -30
- package/docs/specs/meta-ad-creative-media-resolution-spec.md +31 -0
- package/docs/specs/unified-credit-and-scheduled-execution-billing-spec.md +1007 -0
- package/package.json +1 -1
- package/dist/chunk-FMEDPBIV.js.map +0 -1
- package/dist/chunk-HPV4VOQX.js +0 -27
- package/dist/chunk-HPV4VOQX.js.map +0 -1
- package/dist/chunk-RLWCHLV3.js +0 -7
- package/dist/chunk-RLWCHLV3.js.map +0 -1
- package/dist/chunk-RMPPYKUV.js.map +0 -1
- package/dist/extract-bundle-U4D5LW5W.js.map +0 -1
- package/dist/server-MQDCAR6I.js.map +0 -1
- /package/dist/{chunk-RUGJE5EB.js.map → chunk-EJK25QOW.js.map} +0 -0
- /package/dist/{chunk-HE2LQPJ2.js.map → chunk-ZRKFW5FB.js.map} +0 -0
- /package/dist/{db-LIOTIWVN.js.map → db-YHZYG7D2.js.map} +0 -0
- /package/dist/{site-extract-repository-NVSZH35Y.js.map → site-extract-repository-GWKGK46Z.js.map} +0 -0
- /package/dist/{worker-JQTS437L.js.map → worker-O5PZTTPY.js.map} +0 -0
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
// src/lib/media-extractor.ts
|
|
2
|
+
import { createWriteStream, mkdirSync } from "fs";
|
|
3
|
+
import { homedir } from "os";
|
|
4
|
+
import { join, extname, basename } from "path";
|
|
5
|
+
import { pipeline } from "stream/promises";
|
|
6
|
+
import { Readable } from "stream";
|
|
7
|
+
var AD_PATTERNS = [
|
|
8
|
+
"doubleclick.net",
|
|
9
|
+
"googlesyndication.com",
|
|
10
|
+
"googletagmanager.com",
|
|
11
|
+
"google-analytics.com",
|
|
12
|
+
"googletagservices.com",
|
|
13
|
+
"adservice.google",
|
|
14
|
+
"googletag.",
|
|
15
|
+
"pagead2.googlesyndication",
|
|
16
|
+
"facebook.net/tr",
|
|
17
|
+
"connect.facebook.net",
|
|
18
|
+
"fbcdn.net/rsrc",
|
|
19
|
+
"analytics.twitter.com",
|
|
20
|
+
"static.ads-twitter.com",
|
|
21
|
+
"ads.twitter.com",
|
|
22
|
+
"t.co/i/adsct",
|
|
23
|
+
"pixel.advertising.com",
|
|
24
|
+
"hotjar.com",
|
|
25
|
+
"clarity.ms",
|
|
26
|
+
"quantserve.com",
|
|
27
|
+
"scorecardresearch.com",
|
|
28
|
+
"newrelic.com",
|
|
29
|
+
"nr-data.net",
|
|
30
|
+
"segment.io",
|
|
31
|
+
"segment.com",
|
|
32
|
+
"amplitude.com",
|
|
33
|
+
"mixpanel.com",
|
|
34
|
+
"heap.io",
|
|
35
|
+
"fullstory.com",
|
|
36
|
+
"moatads.com",
|
|
37
|
+
"criteo.com",
|
|
38
|
+
"adsrvr.org",
|
|
39
|
+
"rubiconproject.com",
|
|
40
|
+
"pubmatic.com",
|
|
41
|
+
"openx.net",
|
|
42
|
+
"appnexus.com",
|
|
43
|
+
"amazon-adsystem.com",
|
|
44
|
+
"media.net",
|
|
45
|
+
"yieldmo.com",
|
|
46
|
+
"triplelift.com",
|
|
47
|
+
"sharethrough.com",
|
|
48
|
+
"prebid.",
|
|
49
|
+
"smaato.net",
|
|
50
|
+
"indexworm.com",
|
|
51
|
+
"casalemedia.com",
|
|
52
|
+
"outbrain.com",
|
|
53
|
+
"taboola.com",
|
|
54
|
+
"revcontent.com",
|
|
55
|
+
"mgid.com",
|
|
56
|
+
"tawk.to",
|
|
57
|
+
"intercom.io",
|
|
58
|
+
"drift.com",
|
|
59
|
+
"hs-scripts.com",
|
|
60
|
+
"zopim.com",
|
|
61
|
+
"livechatinc.com",
|
|
62
|
+
"userlike.com",
|
|
63
|
+
"onetrust.com",
|
|
64
|
+
"cookielaw.org",
|
|
65
|
+
"cookieinformation.com",
|
|
66
|
+
"trustarc.com",
|
|
67
|
+
"/ads/",
|
|
68
|
+
"/ad/",
|
|
69
|
+
"/banner/",
|
|
70
|
+
"/banners/",
|
|
71
|
+
"/pixel/",
|
|
72
|
+
"/beacon/",
|
|
73
|
+
"/tracking/",
|
|
74
|
+
"/tracker/",
|
|
75
|
+
"/remarketing/",
|
|
76
|
+
"/conversion/",
|
|
77
|
+
"1x1.gif",
|
|
78
|
+
"spacer.gif",
|
|
79
|
+
"blank.gif",
|
|
80
|
+
"transparent.gif"
|
|
81
|
+
];
|
|
82
|
+
var IMAGE_EXTS = /* @__PURE__ */ new Set([".jpg", ".jpeg", ".png", ".webp", ".gif", ".avif", ".svg", ".tiff"]);
|
|
83
|
+
var VIDEO_EXTS = /* @__PURE__ */ new Set([".mp4", ".webm", ".mov", ".avi", ".m4v", ".ogv", ".mkv"]);
|
|
84
|
+
var AUDIO_EXTS = /* @__PURE__ */ new Set([".mp3", ".wav", ".ogg", ".aac", ".m4a", ".flac", ".opus"]);
|
|
85
|
+
function isAdUrl(url) {
|
|
86
|
+
const lower = url.toLowerCase();
|
|
87
|
+
return AD_PATTERNS.some((p) => lower.includes(p));
|
|
88
|
+
}
|
|
89
|
+
function isDataUri(url) {
|
|
90
|
+
return url.startsWith("data:");
|
|
91
|
+
}
|
|
92
|
+
function typeFromUrl(url) {
|
|
93
|
+
try {
|
|
94
|
+
const ext = extname(new URL(url).pathname).toLowerCase();
|
|
95
|
+
if (IMAGE_EXTS.has(ext)) return "image";
|
|
96
|
+
if (VIDEO_EXTS.has(ext)) return "video";
|
|
97
|
+
if (AUDIO_EXTS.has(ext)) return "audio";
|
|
98
|
+
} catch {
|
|
99
|
+
}
|
|
100
|
+
return null;
|
|
101
|
+
}
|
|
102
|
+
function typeFromMime(mime) {
|
|
103
|
+
const lower = mime.toLowerCase();
|
|
104
|
+
if (lower.startsWith("image/")) return "image";
|
|
105
|
+
if (lower.startsWith("video/")) return "video";
|
|
106
|
+
if (lower.startsWith("audio/")) return "audio";
|
|
107
|
+
return null;
|
|
108
|
+
}
|
|
109
|
+
function resolveUrl(raw, base) {
|
|
110
|
+
if (!raw || isDataUri(raw)) return null;
|
|
111
|
+
try {
|
|
112
|
+
return new URL(raw, base).href;
|
|
113
|
+
} catch {
|
|
114
|
+
return null;
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
function safeFilename(url, index) {
|
|
118
|
+
try {
|
|
119
|
+
const u = new URL(url);
|
|
120
|
+
const base = basename(u.pathname).replace(/[^a-zA-Z0-9._-]/g, "_").slice(0, 80);
|
|
121
|
+
return base || `asset-${index}`;
|
|
122
|
+
} catch {
|
|
123
|
+
return `asset-${index}`;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
function extractMediaUrls(html, baseUrl) {
|
|
127
|
+
const seen = /* @__PURE__ */ new Set();
|
|
128
|
+
const urls = [];
|
|
129
|
+
const add = (raw) => {
|
|
130
|
+
const resolved = resolveUrl(raw.trim(), baseUrl);
|
|
131
|
+
if (!resolved || seen.has(resolved) || isAdUrl(resolved)) return;
|
|
132
|
+
seen.add(resolved);
|
|
133
|
+
urls.push(resolved);
|
|
134
|
+
};
|
|
135
|
+
for (const m of html.matchAll(/<img\s[^>]*>/gi)) {
|
|
136
|
+
const tag = m[0];
|
|
137
|
+
for (const attr of ["src", "data-src", "data-lazy-src", "data-original"]) {
|
|
138
|
+
const v = (tag.match(new RegExp(`\\b${attr}\\s*=\\s*["']([^"']+)["']`, "i")) ?? [])[1];
|
|
139
|
+
if (v) add(v);
|
|
140
|
+
}
|
|
141
|
+
const srcset = (tag.match(/\bsrcset\s*=\s*["']([^"']+)["']/i) ?? [])[1];
|
|
142
|
+
if (srcset) {
|
|
143
|
+
for (const part of srcset.split(",")) {
|
|
144
|
+
const u = part.trim().split(/\s+/)[0];
|
|
145
|
+
if (u) add(u);
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
for (const m of html.matchAll(/<source\s[^>]*>/gi)) {
|
|
150
|
+
const tag = m[0];
|
|
151
|
+
const src = (tag.match(/\bsrc\s*=\s*["']([^"']+)["']/i) ?? [])[1];
|
|
152
|
+
if (src) add(src);
|
|
153
|
+
const srcset = (tag.match(/\bsrcset\s*=\s*["']([^"']+)["']/i) ?? [])[1];
|
|
154
|
+
if (srcset) {
|
|
155
|
+
for (const part of srcset.split(",")) {
|
|
156
|
+
const u = part.trim().split(/\s+/)[0];
|
|
157
|
+
if (u) add(u);
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
for (const m of html.matchAll(/<video\s[^>]*>/gi)) {
|
|
162
|
+
const tag = m[0];
|
|
163
|
+
for (const attr of ["src", "poster"]) {
|
|
164
|
+
const v = (tag.match(new RegExp(`\\b${attr}\\s*=\\s*["']([^"']+)["']`, "i")) ?? [])[1];
|
|
165
|
+
if (v) add(v);
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
for (const m of html.matchAll(/<audio\s[^>]*>/gi)) {
|
|
169
|
+
const tag = m[0];
|
|
170
|
+
const v = (tag.match(/\bsrc\s*=\s*["']([^"']+)["']/i) ?? [])[1];
|
|
171
|
+
if (v) add(v);
|
|
172
|
+
}
|
|
173
|
+
for (const m of html.matchAll(/<meta\s[^>]*>/gi)) {
|
|
174
|
+
const tag = m[0];
|
|
175
|
+
const prop = (tag.match(/\b(?:property|name)\s*=\s*["']([^"']+)["']/i) ?? [])[1]?.toLowerCase();
|
|
176
|
+
if (prop === "og:image" || prop === "twitter:image") {
|
|
177
|
+
const content = (tag.match(/\bcontent\s*=\s*["']([^"']+)["']/i) ?? [])[1];
|
|
178
|
+
if (content) add(content);
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
for (const m of html.matchAll(/background(?:-image)?\s*:\s*url\(\s*["']?([^"')]+)["']?\s*\)/gi)) {
|
|
182
|
+
add(m[1]);
|
|
183
|
+
}
|
|
184
|
+
return urls;
|
|
185
|
+
}
|
|
186
|
+
async function downloadAsset(url, destDir, filename) {
|
|
187
|
+
const res = await fetch(url, {
|
|
188
|
+
headers: { "User-Agent": "Mozilla/5.0 (compatible; ThorbitBot/1.0)" },
|
|
189
|
+
signal: AbortSignal.timeout(15e3)
|
|
190
|
+
});
|
|
191
|
+
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
|
192
|
+
if (!res.body) throw new Error("Empty response body");
|
|
193
|
+
const mimeType = res.headers.get("content-type")?.split(";")[0].trim() ?? null;
|
|
194
|
+
let dest = join(destDir, filename);
|
|
195
|
+
if (mimeType && !extname(filename)) {
|
|
196
|
+
const mimeExt = {
|
|
197
|
+
"image/jpeg": ".jpg",
|
|
198
|
+
"image/png": ".png",
|
|
199
|
+
"image/webp": ".webp",
|
|
200
|
+
"image/gif": ".gif",
|
|
201
|
+
"image/svg+xml": ".svg",
|
|
202
|
+
"image/avif": ".avif",
|
|
203
|
+
"video/mp4": ".mp4",
|
|
204
|
+
"video/webm": ".webm",
|
|
205
|
+
"audio/mpeg": ".mp3",
|
|
206
|
+
"audio/ogg": ".ogg",
|
|
207
|
+
"audio/wav": ".wav"
|
|
208
|
+
};
|
|
209
|
+
const ext = mimeExt[mimeType];
|
|
210
|
+
if (ext) dest = dest + ext;
|
|
211
|
+
}
|
|
212
|
+
const writer = createWriteStream(dest);
|
|
213
|
+
await pipeline(Readable.fromWeb(res.body), writer);
|
|
214
|
+
const { statSync } = await import("fs");
|
|
215
|
+
const sizeBytes = statSync(dest).size;
|
|
216
|
+
return { savedPath: dest, sizeBytes, mimeType };
|
|
217
|
+
}
|
|
218
|
+
async function harvestPageMedia(html, pageUrl, options = {}) {
|
|
219
|
+
const types = options.types ?? ["image", "video", "audio"];
|
|
220
|
+
const typesSet = new Set(types);
|
|
221
|
+
const rawUrls = extractMediaUrls(html, pageUrl);
|
|
222
|
+
const totalFound = rawUrls.length;
|
|
223
|
+
const filteredUrls = [];
|
|
224
|
+
const kept = [];
|
|
225
|
+
for (const url of rawUrls) {
|
|
226
|
+
const type = typeFromUrl(url);
|
|
227
|
+
if (!type || !typesSet.has(type)) {
|
|
228
|
+
filteredUrls.push(url);
|
|
229
|
+
continue;
|
|
230
|
+
}
|
|
231
|
+
kept.push({ url, type });
|
|
232
|
+
}
|
|
233
|
+
if (options.outputDir === null) {
|
|
234
|
+
return {
|
|
235
|
+
pageUrl,
|
|
236
|
+
outputDir: null,
|
|
237
|
+
assets: kept.map(({ url, type }, i) => ({
|
|
238
|
+
url,
|
|
239
|
+
type,
|
|
240
|
+
mimeType: null,
|
|
241
|
+
filename: safeFilename(url, i),
|
|
242
|
+
savedPath: null,
|
|
243
|
+
sizeBytes: null
|
|
244
|
+
})),
|
|
245
|
+
filteredCount: filteredUrls.length,
|
|
246
|
+
totalFound
|
|
247
|
+
};
|
|
248
|
+
}
|
|
249
|
+
const domain = (() => {
|
|
250
|
+
try {
|
|
251
|
+
return new URL(pageUrl).hostname.replace(/^www\./, "");
|
|
252
|
+
} catch {
|
|
253
|
+
return "unknown";
|
|
254
|
+
}
|
|
255
|
+
})();
|
|
256
|
+
const stamp = (/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-").slice(0, 19);
|
|
257
|
+
const outDir = options.outputDir ?? join(homedir(), "Downloads", "mcp-scraper", "media", `${stamp}-${domain}`);
|
|
258
|
+
mkdirSync(outDir, { recursive: true });
|
|
259
|
+
const assets = [];
|
|
260
|
+
await Promise.allSettled(
|
|
261
|
+
kept.map(async ({ url, type }, i) => {
|
|
262
|
+
const filename = safeFilename(url, i);
|
|
263
|
+
try {
|
|
264
|
+
const { savedPath, sizeBytes, mimeType } = await downloadAsset(url, outDir, filename);
|
|
265
|
+
const resolvedType = mimeType ? typeFromMime(mimeType) ?? type : type;
|
|
266
|
+
assets.push({ url, type: resolvedType, mimeType, filename: basename(savedPath), savedPath, sizeBytes });
|
|
267
|
+
} catch {
|
|
268
|
+
assets.push({ url, type, mimeType: null, filename, savedPath: null, sizeBytes: null });
|
|
269
|
+
}
|
|
270
|
+
})
|
|
271
|
+
);
|
|
272
|
+
assets.sort((a, b) => {
|
|
273
|
+
if (a.savedPath && !b.savedPath) return -1;
|
|
274
|
+
if (!a.savedPath && b.savedPath) return 1;
|
|
275
|
+
return a.url.localeCompare(b.url);
|
|
276
|
+
});
|
|
277
|
+
return { pageUrl, outputDir: outDir, assets, filteredCount: filteredUrls.length, totalFound };
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
export {
|
|
281
|
+
downloadAsset,
|
|
282
|
+
harvestPageMedia
|
|
283
|
+
};
|
|
284
|
+
//# sourceMappingURL=chunk-JWIE5NCR.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/lib/media-extractor.ts"],"sourcesContent":["import { createWriteStream, mkdirSync } from 'node:fs'\nimport { homedir } from 'node:os'\nimport { join, extname, basename } from 'node:path'\nimport { pipeline } from 'node:stream/promises'\nimport { Readable } from 'node:stream'\n\nconst AD_PATTERNS: string[] = [\n 'doubleclick.net', 'googlesyndication.com', 'googletagmanager.com',\n 'google-analytics.com', 'googletagservices.com', 'adservice.google',\n 'googletag.', 'pagead2.googlesyndication',\n 'facebook.net/tr', 'connect.facebook.net', 'fbcdn.net/rsrc',\n 'analytics.twitter.com', 'static.ads-twitter.com', 'ads.twitter.com',\n 't.co/i/adsct', 'pixel.advertising.com',\n 'hotjar.com', 'clarity.ms', 'quantserve.com', 'scorecardresearch.com',\n 'newrelic.com', 'nr-data.net', 'segment.io', 'segment.com',\n 'amplitude.com', 'mixpanel.com', 'heap.io', 'fullstory.com',\n 'moatads.com', 'criteo.com', 'adsrvr.org', 'rubiconproject.com',\n 'pubmatic.com', 'openx.net', 'appnexus.com', 'amazon-adsystem.com',\n 'media.net', 'yieldmo.com', 'triplelift.com', 'sharethrough.com',\n 'prebid.', 'smaato.net', 'indexworm.com', 'casalemedia.com',\n 'outbrain.com', 'taboola.com', 'revcontent.com', 'mgid.com',\n 'tawk.to', 'intercom.io', 'drift.com', 'hs-scripts.com',\n 'zopim.com', 'livechatinc.com', 'userlike.com',\n 'onetrust.com', 'cookielaw.org', 'cookieinformation.com', 'trustarc.com',\n '/ads/', '/ad/', '/banner/', '/banners/', '/pixel/', '/beacon/',\n '/tracking/', '/tracker/', '/remarketing/', '/conversion/',\n '1x1.gif', 'spacer.gif', 'blank.gif', 'transparent.gif',\n]\n\nconst IMAGE_EXTS = new Set(['.jpg', '.jpeg', '.png', '.webp', '.gif', '.avif', '.svg', '.tiff'])\nconst VIDEO_EXTS = new Set(['.mp4', '.webm', '.mov', '.avi', '.m4v', '.ogv', '.mkv'])\nconst AUDIO_EXTS = new Set(['.mp3', '.wav', '.ogg', '.aac', '.m4a', '.flac', '.opus'])\n\nexport type MediaType = 'image' | 'video' | 'audio'\n\nexport interface MediaAsset {\n url: string\n type: MediaType\n mimeType: string | null\n filename: string\n savedPath: string | null\n sizeBytes: number | null\n}\n\nexport interface MediaManifest {\n pageUrl: string\n outputDir: string | null\n assets: MediaAsset[]\n filteredCount: number\n totalFound: number\n}\n\nexport interface MediaExtractOptions {\n types?: MediaType[]\n outputDir?: string | null\n}\n\nfunction isAdUrl(url: string): boolean {\n const lower = url.toLowerCase()\n return AD_PATTERNS.some(p => lower.includes(p))\n}\n\nfunction isDataUri(url: string): boolean {\n return url.startsWith('data:')\n}\n\nfunction typeFromUrl(url: string): MediaType | null {\n try {\n const ext = extname(new URL(url).pathname).toLowerCase()\n if (IMAGE_EXTS.has(ext)) return 'image'\n if (VIDEO_EXTS.has(ext)) return 'video'\n if (AUDIO_EXTS.has(ext)) return 'audio'\n } catch { /* non-parseable URL */ }\n return null\n}\n\nfunction typeFromMime(mime: string): MediaType | null {\n const lower = mime.toLowerCase()\n if (lower.startsWith('image/')) return 'image'\n if (lower.startsWith('video/')) return 'video'\n if (lower.startsWith('audio/')) return 'audio'\n return null\n}\n\nfunction resolveUrl(raw: string, base: string): string | null {\n if (!raw || isDataUri(raw)) return null\n try { return new URL(raw, base).href } catch { return null }\n}\n\nfunction safeFilename(url: string, index: number): string {\n try {\n const u = new URL(url)\n const base = basename(u.pathname).replace(/[^a-zA-Z0-9._-]/g, '_').slice(0, 80)\n return base || `asset-${index}`\n } catch {\n return `asset-${index}`\n }\n}\n\nexport function extractMediaUrls(html: string, baseUrl: string): string[] {\n const seen = new Set<string>()\n const urls: string[] = []\n\n const add = (raw: string) => {\n const resolved = resolveUrl(raw.trim(), baseUrl)\n if (!resolved || seen.has(resolved) || isAdUrl(resolved)) return\n seen.add(resolved)\n urls.push(resolved)\n }\n\n for (const m of html.matchAll(/<img\\s[^>]*>/gi)) {\n const tag = m[0]\n for (const attr of ['src', 'data-src', 'data-lazy-src', 'data-original']) {\n const v = (tag.match(new RegExp(`\\\\b${attr}\\\\s*=\\\\s*[\"']([^\"']+)[\"']`, 'i')) ?? [])[1]\n if (v) add(v)\n }\n const srcset = (tag.match(/\\bsrcset\\s*=\\s*[\"']([^\"']+)[\"']/i) ?? [])[1]\n if (srcset) {\n for (const part of srcset.split(',')) {\n const u = part.trim().split(/\\s+/)[0]\n if (u) add(u)\n }\n }\n }\n\n for (const m of html.matchAll(/<source\\s[^>]*>/gi)) {\n const tag = m[0]\n const src = (tag.match(/\\bsrc\\s*=\\s*[\"']([^\"']+)[\"']/i) ?? [])[1]\n if (src) add(src)\n const srcset = (tag.match(/\\bsrcset\\s*=\\s*[\"']([^\"']+)[\"']/i) ?? [])[1]\n if (srcset) {\n for (const part of srcset.split(',')) {\n const u = part.trim().split(/\\s+/)[0]\n if (u) add(u)\n }\n }\n }\n\n for (const m of html.matchAll(/<video\\s[^>]*>/gi)) {\n const tag = m[0]\n for (const attr of ['src', 'poster']) {\n const v = (tag.match(new RegExp(`\\\\b${attr}\\\\s*=\\\\s*[\"']([^\"']+)[\"']`, 'i')) ?? [])[1]\n if (v) add(v)\n }\n }\n\n for (const m of html.matchAll(/<audio\\s[^>]*>/gi)) {\n const tag = m[0]\n const v = (tag.match(/\\bsrc\\s*=\\s*[\"']([^\"']+)[\"']/i) ?? [])[1]\n if (v) add(v)\n }\n\n for (const m of html.matchAll(/<meta\\s[^>]*>/gi)) {\n const tag = m[0]\n const prop = (tag.match(/\\b(?:property|name)\\s*=\\s*[\"']([^\"']+)[\"']/i) ?? [])[1]?.toLowerCase()\n if (prop === 'og:image' || prop === 'twitter:image') {\n const content = (tag.match(/\\bcontent\\s*=\\s*[\"']([^\"']+)[\"']/i) ?? [])[1]\n if (content) add(content)\n }\n }\n\n for (const m of html.matchAll(/background(?:-image)?\\s*:\\s*url\\(\\s*[\"']?([^\"')]+)[\"']?\\s*\\)/gi)) {\n add(m[1])\n }\n\n return urls\n}\n\nexport async function downloadAsset(\n url: string,\n destDir: string,\n filename: string,\n): Promise<{ savedPath: string; sizeBytes: number; mimeType: string | null }> {\n const res = await fetch(url, {\n headers: { 'User-Agent': 'Mozilla/5.0 (compatible; ThorbitBot/1.0)' },\n signal: AbortSignal.timeout(15_000),\n })\n if (!res.ok) throw new Error(`HTTP ${res.status}`)\n if (!res.body) throw new Error('Empty response body')\n\n const mimeType = res.headers.get('content-type')?.split(';')[0].trim() ?? null\n\n let dest = join(destDir, filename)\n if (mimeType && !extname(filename)) {\n const mimeExt: Record<string, string> = {\n 'image/jpeg': '.jpg', 'image/png': '.png', 'image/webp': '.webp',\n 'image/gif': '.gif', 'image/svg+xml': '.svg', 'image/avif': '.avif',\n 'video/mp4': '.mp4', 'video/webm': '.webm',\n 'audio/mpeg': '.mp3', 'audio/ogg': '.ogg', 'audio/wav': '.wav',\n }\n const ext = mimeExt[mimeType]\n if (ext) dest = dest + ext\n }\n\n const writer = createWriteStream(dest)\n await pipeline(Readable.fromWeb(res.body as import('stream/web').ReadableStream), writer)\n\n const { statSync } = await import('node:fs')\n const sizeBytes = statSync(dest).size\n\n return { savedPath: dest, sizeBytes, mimeType }\n}\n\nexport async function harvestPageMedia(\n html: string,\n pageUrl: string,\n options: MediaExtractOptions = {},\n): Promise<MediaManifest> {\n const types = options.types ?? ['image', 'video', 'audio']\n const typesSet = new Set(types)\n const rawUrls = extractMediaUrls(html, pageUrl)\n const totalFound = rawUrls.length\n\n const filteredUrls: string[] = []\n const kept: Array<{ url: string; type: MediaType }> = []\n\n for (const url of rawUrls) {\n const type = typeFromUrl(url)\n if (!type || !typesSet.has(type)) { filteredUrls.push(url); continue }\n kept.push({ url, type })\n }\n\n if (options.outputDir === null) {\n return {\n pageUrl,\n outputDir: null,\n assets: kept.map(({ url, type }, i) => ({\n url, type, mimeType: null, filename: safeFilename(url, i), savedPath: null, sizeBytes: null,\n })),\n filteredCount: filteredUrls.length,\n totalFound,\n }\n }\n\n const domain = (() => { try { return new URL(pageUrl).hostname.replace(/^www\\./, '') } catch { return 'unknown' } })()\n const stamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19)\n const outDir = options.outputDir ?? join(homedir(), 'Downloads', 'mcp-scraper', 'media', `${stamp}-${domain}`)\n mkdirSync(outDir, { recursive: true })\n\n const assets: MediaAsset[] = []\n\n await Promise.allSettled(\n kept.map(async ({ url, type }, i) => {\n const filename = safeFilename(url, i)\n try {\n const { savedPath, sizeBytes, mimeType } = await downloadAsset(url, outDir, filename)\n const resolvedType = mimeType ? (typeFromMime(mimeType) ?? type) : type\n assets.push({ url, type: resolvedType, mimeType, filename: basename(savedPath), savedPath, sizeBytes })\n } catch {\n assets.push({ url, type, mimeType: null, filename, savedPath: null, sizeBytes: null })\n }\n })\n )\n\n assets.sort((a, b) => {\n if (a.savedPath && !b.savedPath) return -1\n if (!a.savedPath && b.savedPath) return 1\n return a.url.localeCompare(b.url)\n })\n\n return { pageUrl, outputDir: outDir, assets, filteredCount: filteredUrls.length, totalFound }\n}\n"],"mappings":";AAAA,SAAS,mBAAmB,iBAAiB;AAC7C,SAAS,eAAe;AACxB,SAAS,MAAM,SAAS,gBAAgB;AACxC,SAAS,gBAAgB;AACzB,SAAS,gBAAgB;AAEzB,IAAM,cAAwB;AAAA,EAC5B;AAAA,EAAmB;AAAA,EAAyB;AAAA,EAC5C;AAAA,EAAwB;AAAA,EAAyB;AAAA,EACjD;AAAA,EAAc;AAAA,EACd;AAAA,EAAmB;AAAA,EAAwB;AAAA,EAC3C;AAAA,EAAyB;AAAA,EAA0B;AAAA,EACnD;AAAA,EAAgB;AAAA,EAChB;AAAA,EAAc;AAAA,EAAc;AAAA,EAAkB;AAAA,EAC9C;AAAA,EAAgB;AAAA,EAAe;AAAA,EAAc;AAAA,EAC7C;AAAA,EAAiB;AAAA,EAAgB;AAAA,EAAW;AAAA,EAC5C;AAAA,EAAe;AAAA,EAAc;AAAA,EAAc;AAAA,EAC3C;AAAA,EAAgB;AAAA,EAAa;AAAA,EAAgB;AAAA,EAC7C;AAAA,EAAa;AAAA,EAAe;AAAA,EAAkB;AAAA,EAC9C;AAAA,EAAW;AAAA,EAAc;AAAA,EAAiB;AAAA,EAC1C;AAAA,EAAgB;AAAA,EAAe;AAAA,EAAkB;AAAA,EACjD;AAAA,EAAW;AAAA,EAAe;AAAA,EAAa;AAAA,EACvC;AAAA,EAAa;AAAA,EAAmB;AAAA,EAChC;AAAA,EAAgB;AAAA,EAAiB;AAAA,EAAyB;AAAA,EAC1D;AAAA,EAAS;AAAA,EAAQ;AAAA,EAAY;AAAA,EAAa;AAAA,EAAW;AAAA,EACrD;AAAA,EAAc;AAAA,EAAa;AAAA,EAAiB;AAAA,EAC5C;AAAA,EAAW;AAAA,EAAc;AAAA,EAAa;AACxC;AAEA,IAAM,aAAc,oBAAI,IAAI,CAAC,QAAQ,SAAS,QAAQ,SAAS,QAAQ,SAAS,QAAQ,OAAO,CAAC;AAChG,IAAM,aAAc,oBAAI,IAAI,CAAC,QAAQ,SAAS,QAAQ,QAAQ,QAAQ,QAAQ,MAAM,CAAC;AACrF,IAAM,aAAc,oBAAI,IAAI,CAAC,QAAQ,QAAQ,QAAQ,QAAQ,QAAQ,SAAS,OAAO,CAAC;AA0BtF,SAAS,QAAQ,KAAsB;AACrC,QAAM,QAAQ,IAAI,YAAY;AAC9B,SAAO,YAAY,KAAK,OAAK,MAAM,SAAS,CAAC,CAAC;AAChD;AAEA,SAAS,UAAU,KAAsB;AACvC,SAAO,IAAI,WAAW,OAAO;AAC/B;AAEA,SAAS,YAAY,KAA+B;AAClD,MAAI;AACF,UAAM,MAAM,QAAQ,IAAI,IAAI,GAAG,EAAE,QAAQ,EAAE,YAAY;AACvD,QAAI,WAAW,IAAI,GAAG,EAAG,QAAO;AAChC,QAAI,WAAW,IAAI,GAAG,EAAG,QAAO;AAChC,QAAI,WAAW,IAAI,GAAG,EAAG,QAAO;AAAA,EAClC,QAAQ;AAAA,EAA0B;AAClC,SAAO;AACT;AAEA,SAAS,aAAa,MAAgC;AACpD,QAAM,QAAQ,KAAK,YAAY;AAC/B,MAAI,MAAM,WAAW,QAAQ,EAAG,QAAO;AACvC,MAAI,MAAM,WAAW,QAAQ,EAAG,QAAO;AACvC,MAAI,MAAM,WAAW,QAAQ,EAAG,QAAO;AACvC,SAAO;AACT;AAEA,SAAS,WAAW,KAAa,MAA6B;AAC5D,MAAI,CAAC,OAAO,UAAU,GAAG,EAAG,QAAO;AACnC,MAAI;AAAE,WAAO,IAAI,IAAI,KAAK,IAAI,EAAE;AAAA,EAAK,QAAQ;AAAE,WAAO;AAAA,EAAK;AAC7D;AAEA,SAAS,aAAa,KAAa,OAAuB;AACxD,MAAI;AACF,UAAM,IAAI,IAAI,IAAI,GAAG;AACrB,UAAM,OAAO,SAAS,EAAE,QAAQ,EAAE,QAAQ,oBAAoB,GAAG,EAAE,MAAM,GAAG,EAAE;AAC9E,WAAO,QAAQ,SAAS,KAAK;AAAA,EAC/B,QAAQ;AACN,WAAO,SAAS,KAAK;AAAA,EACvB;AACF;AAEO,SAAS,iBAAiB,MAAc,SAA2B;AACxE,QAAM,OAAO,oBAAI,IAAY;AAC7B,QAAM,OAAiB,CAAC;AAExB,QAAM,MAAM,CAAC,QAAgB;AAC3B,UAAM,WAAW,WAAW,IAAI,KAAK,GAAG,OAAO;AAC/C,QAAI,CAAC,YAAY,KAAK,IAAI,QAAQ,KAAK,QAAQ,QAAQ,EAAG;AAC1D,SAAK,IAAI,QAAQ;AACjB,SAAK,KAAK,QAAQ;AAAA,EACpB;AAEA,aAAW,KAAK,KAAK,SAAS,gBAAgB,GAAG;AAC/C,UAAM,MAAM,EAAE,CAAC;AACf,eAAW,QAAQ,CAAC,OAAO,YAAY,iBAAiB,eAAe,GAAG;AACxE,YAAM,KAAK,IAAI,MAAM,IAAI,OAAO,MAAM,IAAI,6BAA6B,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC;AACrF,UAAI,EAAG,KAAI,CAAC;AAAA,IACd;AACA,UAAM,UAAU,IAAI,MAAM,kCAAkC,KAAK,CAAC,GAAG,CAAC;AACtE,QAAI,QAAQ;AACV,iBAAW,QAAQ,OAAO,MAAM,GAAG,GAAG;AACpC,cAAM,IAAI,KAAK,KAAK,EAAE,MAAM,KAAK,EAAE,CAAC;AACpC,YAAI,EAAG,KAAI,CAAC;AAAA,MACd;AAAA,IACF;AAAA,EACF;AAEA,aAAW,KAAK,KAAK,SAAS,mBAAmB,GAAG;AAClD,UAAM,MAAM,EAAE,CAAC;AACf,UAAM,OAAO,IAAI,MAAM,+BAA+B,KAAK,CAAC,GAAG,CAAC;AAChE,QAAI,IAAK,KAAI,GAAG;AAChB,UAAM,UAAU,IAAI,MAAM,kCAAkC,KAAK,CAAC,GAAG,CAAC;AACtE,QAAI,QAAQ;AACV,iBAAW,QAAQ,OAAO,MAAM,GAAG,GAAG;AACpC,cAAM,IAAI,KAAK,KAAK,EAAE,MAAM,KAAK,EAAE,CAAC;AACpC,YAAI,EAAG,KAAI,CAAC;AAAA,MACd;AAAA,IACF;AAAA,EACF;AAEA,aAAW,KAAK,KAAK,SAAS,kBAAkB,GAAG;AACjD,UAAM,MAAM,EAAE,CAAC;AACf,eAAW,QAAQ,CAAC,OAAO,QAAQ,GAAG;AACpC,YAAM,KAAK,IAAI,MAAM,IAAI,OAAO,MAAM,IAAI,6BAA6B,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC;AACrF,UAAI,EAAG,KAAI,CAAC;AAAA,IACd;AAAA,EACF;AAEA,aAAW,KAAK,KAAK,SAAS,kBAAkB,GAAG;AACjD,UAAM,MAAM,EAAE,CAAC;AACf,UAAM,KAAK,IAAI,MAAM,+BAA+B,KAAK,CAAC,GAAG,CAAC;AAC9D,QAAI,EAAG,KAAI,CAAC;AAAA,EACd;AAEA,aAAW,KAAK,KAAK,SAAS,iBAAiB,GAAG;AAChD,UAAM,MAAM,EAAE,CAAC;AACf,UAAM,QAAQ,IAAI,MAAM,6CAA6C,KAAK,CAAC,GAAG,CAAC,GAAG,YAAY;AAC9F,QAAI,SAAS,cAAc,SAAS,iBAAiB;AACnD,YAAM,WAAW,IAAI,MAAM,mCAAmC,KAAK,CAAC,GAAG,CAAC;AACxE,UAAI,QAAS,KAAI,OAAO;AAAA,IAC1B;AAAA,EACF;AAEA,aAAW,KAAK,KAAK,SAAS,gEAAgE,GAAG;AAC/F,QAAI,EAAE,CAAC,CAAC;AAAA,EACV;AAEA,SAAO;AACT;AAEA,eAAsB,cACpB,KACA,SACA,UAC4E;AAC5E,QAAM,MAAM,MAAM,MAAM,KAAK;AAAA,IAC3B,SAAS,EAAE,cAAc,2CAA2C;AAAA,IACpE,QAAS,YAAY,QAAQ,IAAM;AAAA,EACrC,CAAC;AACD,MAAI,CAAC,IAAI,GAAI,OAAM,IAAI,MAAM,QAAQ,IAAI,MAAM,EAAE;AACjD,MAAI,CAAC,IAAI,KAAM,OAAM,IAAI,MAAM,qBAAqB;AAEpD,QAAM,WAAW,IAAI,QAAQ,IAAI,cAAc,GAAG,MAAM,GAAG,EAAE,CAAC,EAAE,KAAK,KAAK;AAE1E,MAAI,OAAO,KAAK,SAAS,QAAQ;AACjC,MAAI,YAAY,CAAC,QAAQ,QAAQ,GAAG;AAClC,UAAM,UAAkC;AAAA,MACtC,cAAc;AAAA,MAAQ,aAAa;AAAA,MAAQ,cAAc;AAAA,MACzD,aAAa;AAAA,MAAQ,iBAAiB;AAAA,MAAQ,cAAc;AAAA,MAC5D,aAAa;AAAA,MAAQ,cAAc;AAAA,MACnC,cAAc;AAAA,MAAQ,aAAa;AAAA,MAAQ,aAAa;AAAA,IAC1D;AACA,UAAM,MAAM,QAAQ,QAAQ;AAC5B,QAAI,IAAK,QAAO,OAAO;AAAA,EACzB;AAEA,QAAM,SAAS,kBAAkB,IAAI;AACrC,QAAM,SAAS,SAAS,QAAQ,IAAI,IAA2C,GAAG,MAAM;AAExF,QAAM,EAAE,SAAS,IAAI,MAAM,OAAO,IAAS;AAC3C,QAAM,YAAY,SAAS,IAAI,EAAE;AAEjC,SAAO,EAAE,WAAW,MAAM,WAAW,SAAS;AAChD;AAEA,eAAsB,iBACpB,MACA,SACA,UAA+B,CAAC,GACR;AACxB,QAAM,QAAW,QAAQ,SAAS,CAAC,SAAS,SAAS,OAAO;AAC5D,QAAM,WAAW,IAAI,IAAI,KAAK;AAC9B,QAAM,UAAW,iBAAiB,MAAM,OAAO;AAC/C,QAAM,aAAa,QAAQ;AAE3B,QAAM,eAAyB,CAAC;AAChC,QAAM,OAAgD,CAAC;AAEvD,aAAW,OAAO,SAAS;AACzB,UAAM,OAAO,YAAY,GAAG;AAC5B,QAAI,CAAC,QAAQ,CAAC,SAAS,IAAI,IAAI,GAAG;AAAE,mBAAa,KAAK,GAAG;AAAG;AAAA,IAAS;AACrE,SAAK,KAAK,EAAE,KAAK,KAAK,CAAC;AAAA,EACzB;AAEA,MAAI,QAAQ,cAAc,MAAM;AAC9B,WAAO;AAAA,MACL;AAAA,MACA,WAAW;AAAA,MACX,QAAQ,KAAK,IAAI,CAAC,EAAE,KAAK,KAAK,GAAG,OAAO;AAAA,QACtC;AAAA,QAAK;AAAA,QAAM,UAAU;AAAA,QAAM,UAAU,aAAa,KAAK,CAAC;AAAA,QAAG,WAAW;AAAA,QAAM,WAAW;AAAA,MACzF,EAAE;AAAA,MACF,eAAe,aAAa;AAAA,MAC5B;AAAA,IACF;AAAA,EACF;AAEA,QAAM,UAAW,MAAM;AAAE,QAAI;AAAE,aAAO,IAAI,IAAI,OAAO,EAAE,SAAS,QAAQ,UAAU,EAAE;AAAA,IAAE,QAAQ;AAAE,aAAO;AAAA,IAAU;AAAA,EAAE,GAAG;AACtH,QAAM,SAAU,oBAAI,KAAK,GAAE,YAAY,EAAE,QAAQ,SAAS,GAAG,EAAE,MAAM,GAAG,EAAE;AAC1E,QAAM,SAAU,QAAQ,aAAa,KAAK,QAAQ,GAAG,aAAa,eAAe,SAAS,GAAG,KAAK,IAAI,MAAM,EAAE;AAC9G,YAAU,QAAQ,EAAE,WAAW,KAAK,CAAC;AAErC,QAAM,SAAuB,CAAC;AAE9B,QAAM,QAAQ;AAAA,IACZ,KAAK,IAAI,OAAO,EAAE,KAAK,KAAK,GAAG,MAAM;AACnC,YAAM,WAAW,aAAa,KAAK,CAAC;AACpC,UAAI;AACF,cAAM,EAAE,WAAW,WAAW,SAAS,IAAI,MAAM,cAAc,KAAK,QAAQ,QAAQ;AACpF,cAAM,eAAe,WAAY,aAAa,QAAQ,KAAK,OAAQ;AACnE,eAAO,KAAK,EAAE,KAAK,MAAM,cAAc,UAAU,UAAU,SAAS,SAAS,GAAG,WAAW,UAAU,CAAC;AAAA,MACxG,QAAQ;AACN,eAAO,KAAK,EAAE,KAAK,MAAM,UAAU,MAAM,UAAU,WAAW,MAAM,WAAW,KAAK,CAAC;AAAA,MACvF;AAAA,IACF,CAAC;AAAA,EACH;AAEA,SAAO,KAAK,CAAC,GAAG,MAAM;AACpB,QAAI,EAAE,aAAa,CAAC,EAAE,UAAW,QAAO;AACxC,QAAI,CAAC,EAAE,aAAa,EAAE,UAAW,QAAO;AACxC,WAAO,EAAE,IAAI,cAAc,EAAE,GAAG;AAAA,EAClC,CAAC;AAED,SAAO,EAAE,SAAS,WAAW,QAAQ,QAAQ,eAAe,aAAa,QAAQ,WAAW;AAC9F;","names":[]}
|