mcp-scraper 0.64.0 → 0.65.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. package/CHANGELOG.md +16 -0
  2. package/README.md +2 -2
  3. package/dist/bin/api-server.cjs +1028 -497
  4. package/dist/bin/api-server.cjs.map +1 -1
  5. package/dist/bin/api-server.js +2 -2
  6. package/dist/bin/mcp-scraper-cli.cjs +1 -1
  7. package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
  8. package/dist/bin/mcp-scraper-cli.js +1 -1
  9. package/dist/bin/mcp-scraper-install.cjs +2 -2
  10. package/dist/bin/mcp-scraper-install.cjs.map +1 -1
  11. package/dist/bin/mcp-scraper-install.js +2 -2
  12. package/dist/bin/mcp-stdio-server.cjs +49 -9
  13. package/dist/bin/mcp-stdio-server.cjs.map +1 -1
  14. package/dist/bin/mcp-stdio-server.js +4 -4
  15. package/dist/{chunk-TRSZEK2D.js → chunk-2XUXVKT4.js} +3 -3
  16. package/dist/{chunk-TRSZEK2D.js.map → chunk-2XUXVKT4.js.map} +1 -1
  17. package/dist/chunk-5BODYBIP.js +7 -0
  18. package/dist/chunk-5BODYBIP.js.map +1 -0
  19. package/dist/{chunk-BRKCPONM.js → chunk-5PIO7QBG.js} +2 -2
  20. package/dist/{chunk-5QYSY5KI.js → chunk-AMLFKPLL.js} +139 -3
  21. package/dist/chunk-AMLFKPLL.js.map +1 -0
  22. package/dist/{chunk-3NXLGTC6.js → chunk-DCWXVAQT.js} +2 -2
  23. package/dist/{chunk-3NXLGTC6.js.map → chunk-DCWXVAQT.js.map} +1 -1
  24. package/dist/{chunk-EEW3DMI5.js → chunk-PSKRQDGN.js} +50 -10
  25. package/dist/chunk-PSKRQDGN.js.map +1 -0
  26. package/dist/{extract-bundle-FW23CEMG.js → extract-bundle-UWKJT4MU.js} +371 -14
  27. package/dist/extract-bundle-UWKJT4MU.js.map +1 -0
  28. package/dist/{server-TMAWQYZE.js → server-AFEA235I.js} +170 -163
  29. package/dist/server-AFEA235I.js.map +1 -0
  30. package/dist/{site-extract-repository-UXSFD2QM.js → site-extract-repository-J3RH3MOS.js} +3 -3
  31. package/dist/{worker-DD243CON.js → worker-YKRIK5LS.js} +2 -2
  32. package/package.json +3 -1
  33. package/dist/chunk-5QYSY5KI.js.map +0 -1
  34. package/dist/chunk-DYSXI6QU.js +0 -7
  35. package/dist/chunk-DYSXI6QU.js.map +0 -1
  36. package/dist/chunk-EEW3DMI5.js.map +0 -1
  37. package/dist/extract-bundle-FW23CEMG.js.map +0 -1
  38. package/dist/server-TMAWQYZE.js.map +0 -1
  39. /package/dist/{chunk-BRKCPONM.js.map → chunk-5PIO7QBG.js.map} +0 -0
  40. /package/dist/{site-extract-repository-UXSFD2QM.js.map → site-extract-repository-J3RH3MOS.js.map} +0 -0
  41. /package/dist/{worker-DD243CON.js.map → worker-YKRIK5LS.js.map} +0 -0
@@ -0,0 +1 @@
1
+ {"version":3,"sources":["../src/lib/media-extractor.ts","../src/api/extraction-problems.ts","../src/api/commons-embeddings.ts"],"sourcesContent":["import { createWriteStream, mkdirSync, rmSync } from 'node:fs'\nimport { homedir } from 'node:os'\nimport { join, extname, basename } from 'node:path'\nimport { pipeline } from 'node:stream/promises'\nimport { Readable, Transform } from 'node:stream'\nimport { validatePublicHttpUrl } from '../api/url-utils.js'\nimport type { RenderedMediaDiscovery } from './rendered-media.js'\nimport { loadHtmlDocument, type HtmlDocument } from './html-document.js'\n\nconst AD_PATTERNS: string[] = [\n 'doubleclick.net', 'googlesyndication.com', 'googletagmanager.com',\n 'google-analytics.com', 'googletagservices.com', 'adservice.google',\n 'googletag.', 'pagead2.googlesyndication',\n 'facebook.net/tr', 'connect.facebook.net', 'fbcdn.net/rsrc',\n 'analytics.twitter.com', 'static.ads-twitter.com', 'ads.twitter.com',\n 't.co/i/adsct', 'pixel.advertising.com',\n 'hotjar.com', 'clarity.ms', 'quantserve.com', 'scorecardresearch.com',\n 'newrelic.com', 'nr-data.net', 'segment.io', 'segment.com',\n 'amplitude.com', 'mixpanel.com', 'heap.io', 'fullstory.com',\n 'moatads.com', 'criteo.com', 'adsrvr.org', 'rubiconproject.com',\n 'pubmatic.com', 'openx.net', 'appnexus.com', 'amazon-adsystem.com',\n 'media.net', 'yieldmo.com', 'triplelift.com', 'sharethrough.com',\n 'prebid.', 'smaato.net', 'indexworm.com', 'casalemedia.com',\n 'outbrain.com', 'taboola.com', 'revcontent.com', 'mgid.com',\n 'tawk.to', 'intercom.io', 'drift.com', 'hs-scripts.com',\n 'zopim.com', 'livechatinc.com', 'userlike.com',\n 'onetrust.com', 'cookielaw.org', 'cookieinformation.com', 'trustarc.com',\n '/ads/', '/ad/', '/banner/', '/banners/', '/pixel/', '/beacon/',\n '/tracking/', '/tracker/', '/remarketing/', '/conversion/',\n '1x1.gif', 'spacer.gif', 'blank.gif', 'transparent.gif',\n]\n\nconst IMAGE_EXTS = new Set(['.jpg', '.jpeg', '.png', '.webp', '.gif', '.avif', '.svg', '.tiff'])\nconst VIDEO_EXTS = new Set(['.mp4', '.webm', '.mov', '.avi', '.m4v', '.ogv', '.mkv'])\nconst AUDIO_EXTS = new Set(['.mp3', '.wav', '.ogg', '.aac', '.m4a', '.flac', '.opus'])\n\nexport type MediaType = 'image' | 'video' | 'audio'\n\nexport interface MediaAsset {\n url: string\n type: MediaType\n mimeType: string | null\n filename: string\n savedPath: string | null\n sizeBytes: number | null\n discoveryMethods: string[]\n altTexts: string[]\n contexts: string[]\n width: number | null\n height: number | null\n variants: string[]\n finalUrl?: string | null\n duplicateOf?: string | null\n sha256?: string | null\n downloadStatus?: 'downloaded' | 'failed' | 'not_attempted'\n downloadError?: string | null\n inlinePreview?: { data: string; mimeType: string } | null\n}\n\nexport interface MediaManifest {\n pageUrl: string\n outputDir: string | null\n assets: MediaAsset[]\n filteredCount: number\n totalFound: number\n staticFound: number\n renderedFound: number\n completeness: 'complete' | 'partial'\n exhausted: boolean\n stopReason: 'page_exhausted' | 'asset_limit' | 'scroll_round_limit' | 'render_unavailable'\n scrollRounds: number\n warnings: string[]\n artifact: {\n artifactId: string\n filename: string\n contentType: string\n bytes: number\n sha256: string\n expiresAt: string\n downloadUrl: string | null\n downloadUrlExpiresAt: string | null\n localPath: string | null\n } | null\n}\n\nexport interface MediaExtractOptions {\n types?: MediaType[]\n outputDir?: string | null\n rendered?: RenderedMediaDiscovery | null\n maxAssets?: number\n}\n\nexport interface DiscoveredMediaCandidate {\n url: string\n type: MediaType | null\n discoveryMethods: string[]\n altTexts: string[]\n contexts: string[]\n width: number | null\n height: number | null\n}\n\nfunction isAdUrl(url: string): boolean {\n const lower = url.toLowerCase()\n return AD_PATTERNS.some(p => lower.includes(p))\n}\n\nfunction isDataUri(url: string): boolean {\n return url.startsWith('data:')\n}\n\nexport function typeFromUrl(url: string): MediaType | null {\n try {\n const ext = extname(new URL(url).pathname).toLowerCase()\n if (IMAGE_EXTS.has(ext)) return 'image'\n if (VIDEO_EXTS.has(ext)) return 'video'\n if (AUDIO_EXTS.has(ext)) return 'audio'\n } catch { /* non-parseable URL */ }\n return null\n}\n\nexport function typeFromMime(mime: string): MediaType | null {\n const lower = mime.toLowerCase()\n if (lower.startsWith('image/')) return 'image'\n if (lower.startsWith('video/')) return 'video'\n if (lower.startsWith('audio/')) return 'audio'\n return null\n}\n\nfunction resolveUrl(raw: string, base: string): string | null {\n if (!raw || isDataUri(raw)) return null\n try { return new URL(raw, base).href } catch { return null }\n}\n\nfunction safeFilename(url: string, index: number): string {\n try {\n const u = new URL(url)\n const base = basename(u.pathname).replace(/[^a-zA-Z0-9._-]/g, '_').slice(0, 80)\n return base || `asset-${index}`\n } catch {\n return `asset-${index}`\n }\n}\n\nfunction boundedText(value: string | undefined | null, max: number): string | null {\n const normalized = value?.replace(/&(?:amp|#0?38|#x26);/gi, '&').replace(/\\s+/g, ' ').trim()\n return normalized ? normalized.slice(0, max) : null\n}\n\nfunction dimensionsFromUrl(rawUrl: string): { width: number; height: number } | null {\n const match = rawUrl.match(/[-_/](\\d{2,5})x(\\d{2,5})(?=[._/?#-]|$)/i)\n return match ? { width: Number(match[1]), height: Number(match[2]) } : null\n}\n\nexport function mediaFamilyKey(rawUrl: string): string {\n try {\n const url = new URL(rawUrl)\n url.hash = ''\n url.pathname = url.pathname.replace(/-(?:\\d{2,5}x\\d{2,5}|scaled|e\\d{6,})(?=\\.[a-z0-9]{2,6}$)/i, '')\n for (const key of ['w', 'width', 'h', 'height', 'resize']) url.searchParams.delete(key)\n url.searchParams.sort()\n return url.href\n } catch {\n return rawUrl\n }\n}\n\nfunction candidateScore(candidate: DiscoveredMediaCandidate): number {\n const area = (candidate.width ?? 0) * (candidate.height ?? 0)\n const methodBoost = candidate.discoveryMethods.some(method => method.includes('lightbox')) ? 1_000_000_000_000 : 0\n const originalBoost = /-(?:\\d{2,5}x\\d{2,5}|scaled|e\\d{6,})(?=\\.[a-z0-9]{2,6}(?:[?#]|$))/i.test(candidate.url) ? 0 : 100_000_000_000\n return methodBoost + originalBoost + area\n}\n\nfunction mergeCandidate(target: Map<string, DiscoveredMediaCandidate>, incoming: DiscoveredMediaCandidate): void {\n const existing = target.get(incoming.url)\n if (!existing) {\n target.set(incoming.url, incoming)\n return\n }\n existing.discoveryMethods = [...new Set([...existing.discoveryMethods, ...incoming.discoveryMethods])]\n existing.altTexts = [...new Set([...existing.altTexts, ...incoming.altTexts])]\n existing.contexts = [...new Set([...existing.contexts, ...incoming.contexts])]\n if (!existing.type && incoming.type) existing.type = incoming.type\n if ((incoming.width ?? 0) * (incoming.height ?? 0) > (existing.width ?? 0) * (existing.height ?? 0)) {\n existing.width = incoming.width\n existing.height = incoming.height\n }\n}\n\nexport function discoverStaticMedia(html: string | HtmlDocument, baseUrl: string): DiscoveredMediaCandidate[] {\n const found = new Map<string, DiscoveredMediaCandidate>()\n const add = (\n raw: string | undefined,\n method: string,\n explicitType: MediaType | null = null,\n alt?: string | null,\n context?: string | null,\n dimensions?: { width: number; height: number } | null,\n ) => {\n const resolved = raw ? resolveUrl(raw.trim().replace(/&amp;/g, '&'), baseUrl) : null\n if (!resolved || isAdUrl(resolved)) return\n const inferredDimensions = dimensions ?? dimensionsFromUrl(resolved)\n mergeCandidate(found, {\n url: resolved,\n type: explicitType ?? typeFromUrl(resolved),\n discoveryMethods: [method],\n altTexts: alt ? [alt] : [],\n contexts: context ? [context] : [],\n width: inferredDimensions?.width ?? null,\n height: inferredDimensions?.height ?? null,\n })\n }\n const scanSrcset = (value: string | undefined, method: string, type: MediaType, alt?: string | null) => {\n for (const part of (value ?? '').split(',')) {\n const [url, descriptor] = part.trim().split(/\\s+/)\n if (!url) continue\n const width = descriptor?.match(/^(\\d+)w$/)?.[1]\n add(url, method, type, alt, null, width ? { width: Number(width), height: 0 } : null)\n }\n }\n const scan = (document: HtmlDocument, suffix = '') => {\n const $ = document.$\n for (const image of document.images) {\n const attrs = image.attributes\n const alt = boundedText(image.alt, 500)\n const dimensions = image.width && image.height ? { width: image.width, height: image.height } : null\n for (const attr of ['src', 'data-src', 'data-lazy-src', 'data-original', 'data-bg', 'data-background', 'data-bg-url', 'data-lazy-bg', 'data-echo']) {\n const value = attrs[attr]\n add(value, `static-${attr}${suffix}`, 'image', alt, null, dimensions)\n }\n scanSrcset(attrs.srcset, `static-srcset${suffix}`, 'image', alt)\n scanSrcset(attrs['data-srcset'], `static-data-srcset${suffix}`, 'image', alt)\n }\n $('source').each((_index, element) => {\n const node = $(element)\n const declared = node.attr('type') ?? ''\n const type: MediaType = declared.startsWith('video/') ? 'video' : declared.startsWith('audio/') ? 'audio' : 'image'\n add(node.attr('src'), `static-source${suffix}`, type)\n scanSrcset(node.attr('srcset'), `static-source-srcset${suffix}`, type)\n })\n $('video,audio').each((_index, element) => {\n const node = $(element)\n const type = element.tagName.toLowerCase() as 'video' | 'audio'\n add(node.attr('src'), `static-${type}${suffix}`, type)\n if (type === 'video') add(node.attr('poster'), `static-video-poster${suffix}`, 'image')\n })\n $('meta').each((_index, element) => {\n const node = $(element)\n const prop = (node.attr('property') ?? node.attr('name') ?? '').toLowerCase()\n if (prop === 'og:image' || prop === 'og:image:url' || prop === 'twitter:image' || prop === 'twitter:image:src') {\n add(node.attr('content'), `static-${prop}${suffix}`, 'image')\n }\n })\n $('link[href]').each((_index, element) => {\n const node = $(element)\n const rel = (node.attr('rel') ?? '').toLowerCase().split(/\\s+/)\n if (rel.some(value => value === 'icon' || value === 'apple-touch-icon')) {\n add(node.attr('href'), `static-icon${suffix}`, 'image')\n }\n })\n $('svg image').each((_index, element) => {\n const node = $(element)\n add(node.attr('href') ?? node.attr('xlink:href'), `static-svg-image${suffix}`, 'image')\n })\n $('a[href]').has('img').each((_index, element) => {\n const node = $(element)\n const href = node.attr('href')\n if (!href) return\n if (\n /lightbox|gallery|fancybox|glightbox|swipebox|elementor-open-lightbox/i.test(`${node.attr('class') ?? ''} ${node.attr('data-elementor-open-lightbox') ?? ''}`)\n || /\\.(?:avif|gif|jpe?g|png|svg|webp)(?:[?#]|$)/i.test(href)\n ) {\n const alt = boundedText(node.find('img').first().attr('alt'), 500)\n add(href, `static-lightbox-target${suffix}`, 'image', alt)\n }\n })\n const cssSources = $('style').toArray().map(element => $(element).text())\n $('[style]').each((_index, element) => { cssSources.push($(element).attr('style') ?? '') })\n for (const match of cssSources.join('\\n').matchAll(/(?:background(?:-image)?\\s*:\\s*)?url\\(\\s*(?:[\"']|&quot;|&#0?34;|&apos;|&#0?39;)?([^\"')&]+)(?:[\"']|&quot;|&#0?34;|&apos;|&#0?39;)?\\s*\\)/gi)) {\n add(match[1], `static-css-url${suffix}`, 'image')\n }\n for (const match of document.sourceHtml.matchAll(/[\"'](https?:\\/\\/[^\"'\\s<>]+\\.(?:avif|gif|jpe?g|png|svg|webp|mp4|webm|mp3|ogg)(?:\\?[^\"'\\s<>]*)?)[\"']/gi)) {\n add(match[1], `static-bare-url${suffix}`)\n }\n }\n\n const document = typeof html === 'string' ? loadHtmlDocument(html) : html\n scan(document)\n document.$('script').each((_index, element) => {\n const source = document.$(element).text()\n const unescaped = source\n .replace(/\\\\u003[cC]/g, '<').replace(/\\\\u003[eE]/g, '>').replace(/\\\\u0026/gi, '&')\n .replace(/\\\\[\"']/g, match => match.slice(1)).replace(/\\\\\\//g, '/')\n if (unescaped !== source && /<(?:img|source|video|audio)\\b/i.test(unescaped)) {\n scan(loadHtmlDocument(unescaped), '-json-unescaped')\n }\n })\n return [...found.values()]\n}\n\nexport function mergeMediaDiscovery(\n staticAssets: DiscoveredMediaCandidate[],\n rendered: RenderedMediaDiscovery | null | undefined,\n maxAssets = 100,\n): Array<DiscoveredMediaCandidate & { variants: string[] }> {\n const exact = new Map<string, DiscoveredMediaCandidate>()\n for (const asset of staticAssets) mergeCandidate(exact, asset)\n for (const asset of rendered?.assets ?? []) mergeCandidate(exact, { ...asset, type: asset.type })\n\n const families = new Map<string, DiscoveredMediaCandidate[]>()\n for (const asset of exact.values()) {\n const key = mediaFamilyKey(asset.url)\n const family = families.get(key) ?? []\n family.push(asset)\n families.set(key, family)\n }\n return [...families.values()]\n .map(family => {\n const ordered = [...family].sort((a, b) => candidateScore(b) - candidateScore(a) || a.url.localeCompare(b.url))\n const preferred = { ...ordered[0] }\n preferred.discoveryMethods = [...new Set(family.flatMap(asset => asset.discoveryMethods))]\n preferred.altTexts = [...new Set(family.flatMap(asset => asset.altTexts))]\n preferred.contexts = [...new Set(family.flatMap(asset => asset.contexts))]\n return { ...preferred, variants: ordered.slice(1).map(asset => asset.url) }\n })\n .sort((a, b) => candidateScore(b) - candidateScore(a) || a.url.localeCompare(b.url))\n .slice(0, Math.max(1, maxAssets))\n}\n\nexport function extractMediaUrls(html: string, baseUrl: string): string[] {\n return mergeMediaDiscovery(discoverStaticMedia(html, baseUrl), null, 2_000).map(asset => asset.url)\n}\n\nexport async function downloadAsset(\n url: string,\n destDir: string,\n filename: string,\n options: {\n expectedType?: MediaType\n maxBytes?: number\n consumeBytes?: (bytes: number) => boolean\n } = {},\n): Promise<{ savedPath: string; sizeBytes: number; mimeType: string | null }> {\n const maxBytes = Math.max(1, Math.min(options.maxBytes ?? 50 * 1024 * 1024, 100 * 1024 * 1024))\n let target = url\n let res: Response | null = null\n for (let redirects = 0; redirects <= 5; redirects++) {\n const checked = await validatePublicHttpUrl(target, { field: 'media URL' })\n if (checked.error || !checked.parsed) throw new Error(checked.error ?? 'Media URL was rejected')\n res = await fetch(checked.parsed.href, {\n signal: AbortSignal.timeout(15_000),\n redirect: 'manual',\n })\n if (res.status >= 300 && res.status < 400) {\n const location = res.headers.get('location')\n if (!location) throw new Error(`HTTP ${res.status} redirect did not include Location`)\n target = new URL(location, checked.parsed.href).href\n res = null\n continue\n }\n break\n }\n if (!res) throw new Error('Media download exceeded five redirects')\n if (!res.ok) throw new Error(`HTTP ${res.status}`)\n if (!res.body) throw new Error('Empty response body')\n\n const mimeType = res.headers.get('content-type')?.split(';')[0].trim() ?? null\n if (options.expectedType && (!mimeType || typeFromMime(mimeType) !== options.expectedType)) {\n throw new Error(`Expected ${options.expectedType} content but received ${mimeType ?? 'no content-type'}`)\n }\n const declaredBytes = Number(res.headers.get('content-length') ?? 0)\n if (Number.isFinite(declaredBytes) && declaredBytes > maxBytes) {\n throw new Error(`Media asset exceeds ${maxBytes} byte limit`)\n }\n\n let dest = join(destDir, filename)\n if (mimeType && !extname(filename)) {\n const mimeExt: Record<string, string> = {\n 'image/jpeg': '.jpg', 'image/png': '.png', 'image/webp': '.webp',\n 'image/gif': '.gif', 'image/svg+xml': '.svg', 'image/avif': '.avif',\n 'video/mp4': '.mp4', 'video/webm': '.webm',\n 'audio/mpeg': '.mp3', 'audio/ogg': '.ogg', 'audio/wav': '.wav',\n }\n const ext = mimeExt[mimeType]\n if (ext) dest = dest + ext\n }\n\n const writer = createWriteStream(dest)\n await new Promise<void>((resolve, reject) => {\n writer.once('open', () => resolve())\n writer.once('error', reject)\n })\n let streamedBytes = 0\n const limiter = new Transform({\n transform(chunk: Buffer, _encoding, callback) {\n const bytes = chunk.length\n if (streamedBytes + bytes > maxBytes) {\n callback(new Error(`Media asset exceeds ${maxBytes} byte limit`))\n return\n }\n if (options.consumeBytes && !options.consumeBytes(bytes)) {\n callback(new Error('Media export total byte limit exceeded'))\n return\n }\n streamedBytes += bytes\n callback(null, chunk)\n },\n })\n try {\n await pipeline(Readable.fromWeb(res.body as import('stream/web').ReadableStream), limiter, writer)\n } catch (error) {\n writer.destroy()\n rmSync(dest, { force: true })\n throw error\n }\n\n const { statSync } = await import('node:fs')\n const sizeBytes = statSync(dest).size\n\n return { savedPath: dest, sizeBytes, mimeType }\n}\n\nexport async function harvestPageMedia(\n html: string,\n pageUrl: string,\n options: MediaExtractOptions = {},\n): Promise<MediaManifest> {\n const types = options.types ?? ['image', 'video', 'audio']\n const typesSet = new Set(types)\n const staticAssets = discoverStaticMedia(html, pageUrl)\n const maxAssets = Math.max(1, Math.min(options.maxAssets ?? 100, 250))\n const allDiscovered = mergeMediaDiscovery(staticAssets, options.rendered, 2_000)\n const discovered = allDiscovered.slice(0, maxAssets)\n const totalFound = new Set([\n ...staticAssets.map(asset => asset.url),\n ...(options.rendered?.assets ?? []).map(asset => asset.url),\n ]).size\n const kept = discovered.flatMap(asset => {\n const type = asset.type ?? typeFromUrl(asset.url)\n return type && typesSet.has(type) ? [{ ...asset, type }] : []\n })\n const filteredCount = totalFound - kept.length\n const retentionLimitReached = allDiscovered.length > maxAssets\n const exhausted = options.rendered?.exhausted === true && !retentionLimitReached\n const stopReason = retentionLimitReached ? 'asset_limit' : options.rendered?.stopReason ?? 'render_unavailable'\n const warnings = retentionLimitReached\n ? [`Media discovery found ${allDiscovered.length} responsive families; only the requested ${maxAssets} records were retained.`]\n : options.rendered\n ? options.rendered.exhausted ? [] : [`Rendered media discovery stopped with ${options.rendered.stopReason}; the inventory may be incomplete.`]\n : ['Rendered media discovery was unavailable; this manifest contains static-source evidence only.']\n\n const baseManifest: Omit<MediaManifest, 'outputDir' | 'assets'> = {\n pageUrl,\n staticFound: staticAssets.length,\n renderedFound: options.rendered?.assets.length ?? 0,\n filteredCount,\n totalFound,\n completeness: exhausted ? 'complete' as const : 'partial' as const,\n exhausted,\n stopReason,\n scrollRounds: options.rendered?.scrollRounds ?? 0,\n warnings,\n artifact: null,\n }\n\n if (options.outputDir === null) {\n return {\n ...baseManifest,\n outputDir: null,\n assets: kept.map((asset, i) => ({\n ...asset,\n mimeType: null, filename: safeFilename(asset.url, i), savedPath: null, sizeBytes: null,\n sha256: null, downloadStatus: 'not_attempted' as const, downloadError: null, inlinePreview: null,\n })),\n }\n }\n\n const domain = (() => { try { return new URL(pageUrl).hostname.replace(/^www\\./, '') } catch { return 'unknown' } })()\n const stamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19)\n const outDir = options.outputDir ?? join(homedir(), 'Downloads', 'mcp-scraper', 'media', `${stamp}-${domain}`)\n mkdirSync(outDir, { recursive: true })\n\n const assets: MediaAsset[] = []\n\n await Promise.allSettled(\n kept.map(async (asset, i) => {\n const filename = safeFilename(asset.url, i)\n try {\n const { savedPath, sizeBytes, mimeType } = await downloadAsset(asset.url, outDir, filename)\n const resolvedType = mimeType ? (typeFromMime(mimeType) ?? asset.type) : asset.type\n assets.push({\n ...asset, type: resolvedType, mimeType, filename: basename(savedPath), savedPath, sizeBytes,\n sha256: null, downloadStatus: 'downloaded', downloadError: null, inlinePreview: null,\n })\n } catch (error) {\n assets.push({\n ...asset, mimeType: null, filename, savedPath: null, sizeBytes: null, sha256: null,\n downloadStatus: 'failed', downloadError: error instanceof Error ? error.message : 'media_download_failed', inlinePreview: null,\n })\n }\n })\n )\n\n assets.sort((a, b) => {\n if (a.savedPath && !b.savedPath) return -1\n if (!a.savedPath && b.savedPath) return 1\n return a.url.localeCompare(b.url)\n })\n\n return { ...baseManifest, outputDir: outDir, assets }\n}\n","import {\n buildPublicErrorEnvelope,\n type PublicChargeStatus,\n type PublicErrorDetails,\n type PublicErrorEnvelope,\n} from '../errors.js'\nimport { isVendorUnavailableError } from './vendor-errors.js'\n\nexport type ExtractionErrorCode =\n | 'vendor_unavailable'\n | 'page_not_found'\n | 'page_forbidden'\n | 'page_rate_limited'\n | 'page_server_error'\n | 'page_http_error'\n | 'bot_check_unresolved'\n | 'page_too_large'\n | 'page_unreachable'\n | 'browser_session_interrupted'\n | 'extraction_failed'\n\nexport interface ExtractionProblem {\n errorCode: ExtractionErrorCode\n httpStatus: number | null\n retryable: boolean\n}\n\nexport interface ExtractionErrorContext {\n chargeStatus?: PublicChargeStatus\n details?: PublicErrorDetails\n}\n\nexport type ExtractionErrorResponse = PublicErrorEnvelope & { error: string }\n\nconst UNREACHABLE_CODES = new Set([\n 'browser_navigation_blocked',\n 'browser_navigation_failed',\n 'browser_timeout',\n 'browser_session_failed',\n 'browser_result_missing',\n])\n\nfunction errorCode(err: unknown): string | undefined {\n return err instanceof Error ? (err as Error & { code?: string }).code : undefined\n}\n\nfunction errorMessage(err: unknown): string {\n return err instanceof Error ? err.message : String(err)\n}\n\nfunction httpStatusFromBrowserHttpCode(value: string | undefined): number | null {\n const match = value?.match(/^browser_http_(\\d+)$/)\n return match ? Number(match[1]) : null\n}\n\nfunction bucketHttpStatus(status: number): ExtractionErrorCode {\n if (status === 404) return 'page_not_found'\n if (status === 403) return 'page_forbidden'\n if (status === 429) return 'page_rate_limited'\n if (status >= 500 && status <= 599) return 'page_server_error'\n return 'page_http_error'\n}\n\nexport function classifyExtractionProblem(err: unknown): ExtractionProblem {\n const message = errorMessage(err)\n const code = errorCode(err)\n\n if (isVendorUnavailableError(err, message)) {\n return { errorCode: 'vendor_unavailable', httpStatus: null, retryable: true }\n }\n\n const httpStatus = httpStatusFromBrowserHttpCode(code) ?? httpStatusFromBrowserHttpCode(message.match(/^(browser_http_\\d+)/)?.[1])\n if (httpStatus != null) {\n return { errorCode: bucketHttpStatus(httpStatus), httpStatus, retryable: httpStatus === 429 || httpStatus >= 500 }\n }\n\n if (code === 'browser_challenge_unresolved' || /^browser_challenge_unresolved/.test(message)) {\n return { errorCode: 'bot_check_unresolved', httpStatus: null, retryable: true }\n }\n\n if (code === 'browser_response_too_large' || /exceeds \\d+ byte limit/i.test(message)) {\n return { errorCode: 'page_too_large', httpStatus: null, retryable: false }\n }\n\n if (/browser (?:has been )?closed|context closed|target page, context or browser has been closed|session closed/i.test(message)) {\n return { errorCode: 'browser_session_interrupted', httpStatus: null, retryable: true }\n }\n\n if (UNREACHABLE_CODES.has(code ?? '') || /timeout|navigation_failed|ERR_/i.test(message)) {\n return { errorCode: 'page_unreachable', httpStatus: null, retryable: true }\n }\n\n return { errorCode: 'extraction_failed', httpStatus: null, retryable: false }\n}\n\nexport function publicExtractionErrorMessage(code: ExtractionErrorCode, httpStatus: number | null): string {\n switch (code) {\n case 'page_not_found':\n return `The page could not be found (HTTP ${httpStatus ?? 404}). It may have been moved or deleted.`\n case 'page_forbidden':\n return `The site refused the request (HTTP ${httpStatus ?? 403}).`\n case 'page_rate_limited':\n return 'The site is rate-limiting requests right now (HTTP 429). Retrying after a short wait usually works.'\n case 'page_server_error':\n return `The page's own server returned an error (HTTP ${httpStatus}). This is on the target site's side, not something a retry here can fix.`\n case 'page_http_error':\n return `The page returned an unexpected HTTP ${httpStatus} response.`\n case 'bot_check_unresolved':\n return 'This site has automated-traffic protection (a bot/CAPTCHA check) that could not be resolved in time. Some sites are simply not extractable this way.'\n case 'page_too_large':\n return 'The page is larger than we can safely process.'\n case 'page_unreachable':\n return 'The page did not respond in time, or the connection could not be completed.'\n case 'browser_session_interrupted':\n return 'The browser session closed before extraction completed. A retry uses a fresh browser session.'\n case 'vendor_unavailable':\n return \"MCP Scraper's internal browser services require MCP Scraper team attention. Servers or IPs are down until this is fixed — this is not caused by your request. Please retry in a few minutes.\"\n case 'extraction_failed':\n return 'The page could not be extracted. Please retry, or contact support if this persists for the same URL.'\n }\n}\n\nexport function extractionProblemResponse(problem: ExtractionProblem, context: ExtractionErrorContext = {}): ExtractionErrorResponse {\n const message = publicExtractionErrorMessage(problem.errorCode, problem.httpStatus)\n const envelope = buildPublicErrorEnvelope({\n errorCode: problem.errorCode,\n retryable: problem.retryable,\n chargeStatus: context.chargeStatus,\n details: {\n ...(context.details ?? {}),\n ...(problem.httpStatus != null ? { http_status: problem.httpStatus } : {}),\n },\n })\n return { ...envelope, message, error: message }\n}\n\nexport function extractionWireStatus(problem: ExtractionProblem): 403 | 404 | 429 | 500 | 502 | 503 {\n if (problem.httpStatus === 404) return 404\n if (problem.httpStatus === 403) return 403\n if (problem.httpStatus === 429) return 429\n if (problem.httpStatus != null && problem.httpStatus >= 500 && problem.httpStatus <= 599) return 500\n if (problem.httpStatus != null) return 502\n if (problem.errorCode === 'vendor_unavailable') return 503\n return 502\n}\n\nconst WAYBACK_CAPTURE_MISSING = 'wayback_capture_missing'\n\nexport function publicizeExtractionFailure(\n failureCode: string | null | undefined,\n failureReason: string | null | undefined,\n): { failureCode: string | null; failureReason: string | null } {\n if (failureCode == null) return { failureCode: null, failureReason: null }\n if (failureCode === WAYBACK_CAPTURE_MISSING) return { failureCode, failureReason: failureReason ?? null }\n const err = new Error(failureReason ?? failureCode) as Error & { code?: string }\n err.code = failureCode\n const problem = classifyExtractionProblem(err)\n return { failureCode: problem.errorCode, failureReason: publicExtractionErrorMessage(problem.errorCode, problem.httpStatus) }\n}\n","import { neon, type NeonQueryFunction } from '@neondatabase/serverless'\nimport { getDb } from './db.js'\n\nexport const COMMONS_EMBED_PROVIDER = 'jina'\n\nlet _vectorSql: NeonQueryFunction<false, false> | null = null\nlet vectorSchemaReady = false\n\nexport function commonsEmbedModel(): string {\n return (process.env.JINA_EMBED_MODEL ?? 'jina-embeddings-v5-omni-small').trim()\n}\n\nexport function commonsEmbedDim(): number {\n return Number((process.env.JINA_EMBED_DIM ?? '1024').trim())\n}\n\nexport function commonsSemanticSearchConfigured(): boolean {\n return Boolean(process.env.JINA_API_KEY?.trim() && process.env.MEMORY_DATABASE_URL?.trim())\n}\n\nfunction vectorSql(): NeonQueryFunction<false, false> {\n if (_vectorSql) return _vectorSql\n const url = process.env.MEMORY_DATABASE_URL?.trim()\n if (!url) throw new Error('MEMORY_DATABASE_URL is not set; Commons semantic search needs the shared Postgres.')\n _vectorSql = neon(url)\n return _vectorSql\n}\n\nexport function resetCommonsVectorSchemaForTesting(): void {\n vectorSchemaReady = false\n _vectorSql = null\n}\n\nexport async function ensureCommonsVectorSchema(): Promise<void> {\n if (vectorSchemaReady) return\n const dimension = commonsEmbedDim()\n await vectorSql().query('CREATE EXTENSION IF NOT EXISTS vector')\n await vectorSql().query(`\n CREATE TABLE IF NOT EXISTS commons_index_vectors (\n document_id TEXT PRIMARY KEY,\n entity_id TEXT NOT NULL,\n document_type TEXT NOT NULL,\n title TEXT NOT NULL,\n embedding vector(${dimension}) NOT NULL,\n model TEXT NOT NULL,\n updated_at TIMESTAMPTZ NOT NULL DEFAULT now()\n )\n `)\n await vectorSql().query('CREATE INDEX IF NOT EXISTS commons_index_vectors_entity ON commons_index_vectors(entity_id)')\n vectorSchemaReady = true\n}\n\nexport async function embedCommonsTexts(texts: string[]): Promise<number[][]> {\n const apiKey = process.env.JINA_API_KEY?.trim()\n if (!apiKey) throw new Error('JINA_API_KEY is not set; Commons semantic search cannot embed.')\n if (!texts.length) return []\n const response = await fetch('https://api.jina.ai/v1/embeddings', {\n method: 'POST',\n headers: { authorization: `Bearer ${apiKey}`, 'content-type': 'application/json' },\n body: JSON.stringify({\n model: commonsEmbedModel(),\n dimensions: commonsEmbedDim(),\n input: texts.map(text => ({ text: text.slice(0, 8_000) })),\n }),\n signal: AbortSignal.timeout(60_000),\n })\n if (!response.ok) {\n throw new Error(`Jina embedding request failed with HTTP ${response.status}: ${(await response.text()).slice(0, 200)}`)\n }\n const payload = await response.json() as { data?: Array<{ embedding: number[] }> }\n const vectors = (payload.data ?? []).map(item => item.embedding)\n if (vectors.length !== texts.length) {\n throw new Error(`Jina returned ${vectors.length} embeddings for ${texts.length} inputs.`)\n }\n return vectors\n}\n\nexport async function embedQueuedCommonsDocuments(limit = 50): Promise<{\n claimed: number\n embedded: number\n failed: number\n remaining: number\n}> {\n if (!commonsSemanticSearchConfigured()) return { claimed: 0, embedded: 0, failed: 0, remaining: 0 }\n await ensureCommonsVectorSchema()\n const bounded = Math.max(1, Math.min(200, Math.floor(limit)))\n const queued = await getDb().execute({\n sql: `SELECT id, entity_id, document_type, title, text FROM commons_index_documents\n WHERE embedding_status IN ('queued', 'failed') ORDER BY updated_at ASC LIMIT ?`,\n args: [bounded],\n })\n const rows = queued.rows as unknown as Array<{ id: string; entity_id: string; document_type: string; title: string; text: string }>\n if (!rows.length) return { claimed: 0, embedded: 0, failed: 0, remaining: await queuedCommonsDocumentCount() }\n\n let embedded = 0\n let failed = 0\n const model = commonsEmbedModel()\n try {\n const vectors = await embedCommonsTexts(rows.map(row => `${row.title}\\n\\n${row.text}`))\n for (const [index, row] of rows.entries()) {\n const literal = `[${vectors[index].join(',')}]`\n await vectorSql().query(\n `INSERT INTO commons_index_vectors (document_id, entity_id, document_type, title, embedding, model, updated_at)\n VALUES ($1, $2, $3, $4, $5::vector, $6, now())\n ON CONFLICT (document_id) DO UPDATE SET entity_id = EXCLUDED.entity_id, document_type = EXCLUDED.document_type,\n title = EXCLUDED.title, embedding = EXCLUDED.embedding, model = EXCLUDED.model, updated_at = now()`,\n [row.id, row.entity_id, row.document_type, row.title, literal, model],\n )\n await getDb().execute({\n sql: `UPDATE commons_index_documents SET embedding_status = 'indexed', embedding_provider = ?, embedding_model = ?,\n vector_ref = ?, indexed_at = ?, error = NULL WHERE id = ?`,\n args: [COMMONS_EMBED_PROVIDER, model, row.id, new Date().toISOString(), row.id],\n })\n embedded += 1\n }\n } catch (error) {\n failed = rows.length - embedded\n const message = (error instanceof Error ? error.message : String(error)).slice(0, 500)\n for (const row of rows.slice(embedded)) {\n await getDb().execute({\n sql: `UPDATE commons_index_documents SET embedding_status = 'failed', error = ? WHERE id = ?`,\n args: [message, row.id],\n }).catch(() => undefined)\n }\n }\n return { claimed: rows.length, embedded, failed, remaining: await queuedCommonsDocumentCount() }\n}\n\nexport async function queuedCommonsDocumentCount(): Promise<number> {\n const result = await getDb().execute(`SELECT COUNT(*) AS n FROM commons_index_documents WHERE embedding_status IN ('queued', 'failed')`)\n return Number((result.rows[0] as unknown as { n: number })?.n ?? 0)\n}\n\nexport async function semanticCommonsEntityScores(query: string, limit = 40): Promise<Map<string, number>> {\n const scores = new Map<string, number>()\n if (!commonsSemanticSearchConfigured() || !query.trim()) return scores\n await ensureCommonsVectorSchema()\n const [vector] = await embedCommonsTexts([query])\n if (!vector) return scores\n const rows = await vectorSql().query(\n `SELECT entity_id, MAX(1 - (embedding <=> $1::vector)) AS score\n FROM commons_index_vectors GROUP BY entity_id ORDER BY score DESC LIMIT $2`,\n [`[${vector.join(',')}]`, Math.max(1, Math.min(100, limit))],\n ) as unknown as Array<{ entity_id: string; score: number }>\n for (const row of rows) scores.set(String(row.entity_id), Number(row.score))\n return scores\n}\n"],"mappings":";;;;;;;;;;;;;;;AAAA,SAAS,mBAAmB,WAAW,cAAc;AACrD,SAAS,eAAe;AACxB,SAAS,MAAM,SAAS,gBAAgB;AACxC,SAAS,gBAAgB;AACzB,SAAS,UAAU,iBAAiB;AAKpC,IAAM,cAAwB;AAAA,EAC5B;AAAA,EAAmB;AAAA,EAAyB;AAAA,EAC5C;AAAA,EAAwB;AAAA,EAAyB;AAAA,EACjD;AAAA,EAAc;AAAA,EACd;AAAA,EAAmB;AAAA,EAAwB;AAAA,EAC3C;AAAA,EAAyB;AAAA,EAA0B;AAAA,EACnD;AAAA,EAAgB;AAAA,EAChB;AAAA,EAAc;AAAA,EAAc;AAAA,EAAkB;AAAA,EAC9C;AAAA,EAAgB;AAAA,EAAe;AAAA,EAAc;AAAA,EAC7C;AAAA,EAAiB;AAAA,EAAgB;AAAA,EAAW;AAAA,EAC5C;AAAA,EAAe;AAAA,EAAc;AAAA,EAAc;AAAA,EAC3C;AAAA,EAAgB;AAAA,EAAa;AAAA,EAAgB;AAAA,EAC7C;AAAA,EAAa;AAAA,EAAe;AAAA,EAAkB;AAAA,EAC9C;AAAA,EAAW;AAAA,EAAc;AAAA,EAAiB;AAAA,EAC1C;AAAA,EAAgB;AAAA,EAAe;AAAA,EAAkB;AAAA,EACjD;AAAA,EAAW;AAAA,EAAe;AAAA,EAAa;AAAA,EACvC;AAAA,EAAa;AAAA,EAAmB;AAAA,EAChC;AAAA,EAAgB;AAAA,EAAiB;AAAA,EAAyB;AAAA,EAC1D;AAAA,EAAS;AAAA,EAAQ;AAAA,EAAY;AAAA,EAAa;AAAA,EAAW;AAAA,EACrD;AAAA,EAAc;AAAA,EAAa;AAAA,EAAiB;AAAA,EAC5C;AAAA,EAAW;AAAA,EAAc;AAAA,EAAa;AACxC;AAEA,IAAM,aAAc,oBAAI,IAAI,CAAC,QAAQ,SAAS,QAAQ,SAAS,QAAQ,SAAS,QAAQ,OAAO,CAAC;AAChG,IAAM,aAAc,oBAAI,IAAI,CAAC,QAAQ,SAAS,QAAQ,QAAQ,QAAQ,QAAQ,MAAM,CAAC;AACrF,IAAM,aAAc,oBAAI,IAAI,CAAC,QAAQ,QAAQ,QAAQ,QAAQ,QAAQ,SAAS,OAAO,CAAC;AAoEtF,SAAS,QAAQ,KAAsB;AACrC,QAAM,QAAQ,IAAI,YAAY;AAC9B,SAAO,YAAY,KAAK,OAAK,MAAM,SAAS,CAAC,CAAC;AAChD;AAEA,SAAS,UAAU,KAAsB;AACvC,SAAO,IAAI,WAAW,OAAO;AAC/B;AAEO,SAAS,YAAY,KAA+B;AACzD,MAAI;AACF,UAAM,MAAM,QAAQ,IAAI,IAAI,GAAG,EAAE,QAAQ,EAAE,YAAY;AACvD,QAAI,WAAW,IAAI,GAAG,EAAG,QAAO;AAChC,QAAI,WAAW,IAAI,GAAG,EAAG,QAAO;AAChC,QAAI,WAAW,IAAI,GAAG,EAAG,QAAO;AAAA,EAClC,QAAQ;AAAA,EAA0B;AAClC,SAAO;AACT;AAEO,SAAS,aAAa,MAAgC;AAC3D,QAAM,QAAQ,KAAK,YAAY;AAC/B,MAAI,MAAM,WAAW,QAAQ,EAAG,QAAO;AACvC,MAAI,MAAM,WAAW,QAAQ,EAAG,QAAO;AACvC,MAAI,MAAM,WAAW,QAAQ,EAAG,QAAO;AACvC,SAAO;AACT;AAEA,SAAS,WAAW,KAAa,MAA6B;AAC5D,MAAI,CAAC,OAAO,UAAU,GAAG,EAAG,QAAO;AACnC,MAAI;AAAE,WAAO,IAAI,IAAI,KAAK,IAAI,EAAE;AAAA,EAAK,QAAQ;AAAE,WAAO;AAAA,EAAK;AAC7D;AAEA,SAAS,aAAa,KAAa,OAAuB;AACxD,MAAI;AACF,UAAM,IAAI,IAAI,IAAI,GAAG;AACrB,UAAM,OAAO,SAAS,EAAE,QAAQ,EAAE,QAAQ,oBAAoB,GAAG,EAAE,MAAM,GAAG,EAAE;AAC9E,WAAO,QAAQ,SAAS,KAAK;AAAA,EAC/B,QAAQ;AACN,WAAO,SAAS,KAAK;AAAA,EACvB;AACF;AAEA,SAAS,YAAY,OAAkC,KAA4B;AACjF,QAAM,aAAa,OAAO,QAAQ,0BAA0B,GAAG,EAAE,QAAQ,QAAQ,GAAG,EAAE,KAAK;AAC3F,SAAO,aAAa,WAAW,MAAM,GAAG,GAAG,IAAI;AACjD;AAEA,SAAS,kBAAkB,QAA0D;AACnF,QAAM,QAAQ,OAAO,MAAM,yCAAyC;AACpE,SAAO,QAAQ,EAAE,OAAO,OAAO,MAAM,CAAC,CAAC,GAAG,QAAQ,OAAO,MAAM,CAAC,CAAC,EAAE,IAAI;AACzE;AAEO,SAAS,eAAe,QAAwB;AACrD,MAAI;AACF,UAAM,MAAM,IAAI,IAAI,MAAM;AAC1B,QAAI,OAAO;AACX,QAAI,WAAW,IAAI,SAAS,QAAQ,4DAA4D,EAAE;AAClG,eAAW,OAAO,CAAC,KAAK,SAAS,KAAK,UAAU,QAAQ,EAAG,KAAI,aAAa,OAAO,GAAG;AACtF,QAAI,aAAa,KAAK;AACtB,WAAO,IAAI;AAAA,EACb,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEA,SAAS,eAAe,WAA6C;AACnE,QAAM,QAAQ,UAAU,SAAS,MAAM,UAAU,UAAU;AAC3D,QAAM,cAAc,UAAU,iBAAiB,KAAK,YAAU,OAAO,SAAS,UAAU,CAAC,IAAI,OAAoB;AACjH,QAAM,gBAAgB,oEAAoE,KAAK,UAAU,GAAG,IAAI,IAAI;AACpH,SAAO,cAAc,gBAAgB;AACvC;AAEA,SAAS,eAAe,QAA+C,UAA0C;AAC/G,QAAM,WAAW,OAAO,IAAI,SAAS,GAAG;AACxC,MAAI,CAAC,UAAU;AACb,WAAO,IAAI,SAAS,KAAK,QAAQ;AACjC;AAAA,EACF;AACA,WAAS,mBAAmB,CAAC,GAAG,oBAAI,IAAI,CAAC,GAAG,SAAS,kBAAkB,GAAG,SAAS,gBAAgB,CAAC,CAAC;AACrG,WAAS,WAAW,CAAC,GAAG,oBAAI,IAAI,CAAC,GAAG,SAAS,UAAU,GAAG,SAAS,QAAQ,CAAC,CAAC;AAC7E,WAAS,WAAW,CAAC,GAAG,oBAAI,IAAI,CAAC,GAAG,SAAS,UAAU,GAAG,SAAS,QAAQ,CAAC,CAAC;AAC7E,MAAI,CAAC,SAAS,QAAQ,SAAS,KAAM,UAAS,OAAO,SAAS;AAC9D,OAAK,SAAS,SAAS,MAAM,SAAS,UAAU,MAAM,SAAS,SAAS,MAAM,SAAS,UAAU,IAAI;AACnG,aAAS,QAAQ,SAAS;AAC1B,aAAS,SAAS,SAAS;AAAA,EAC7B;AACF;AAEO,SAAS,oBAAoB,MAA6B,SAA6C;AAC5G,QAAM,QAAQ,oBAAI,IAAsC;AACxD,QAAM,MAAM,CACV,KACA,QACA,eAAiC,MACjC,KACA,SACA,eACG;AACH,UAAM,WAAW,MAAM,WAAW,IAAI,KAAK,EAAE,QAAQ,UAAU,GAAG,GAAG,OAAO,IAAI;AAChF,QAAI,CAAC,YAAY,QAAQ,QAAQ,EAAG;AACpC,UAAM,qBAAqB,cAAc,kBAAkB,QAAQ;AACnE,mBAAe,OAAO;AAAA,MACpB,KAAK;AAAA,MACL,MAAM,gBAAgB,YAAY,QAAQ;AAAA,MAC1C,kBAAkB,CAAC,MAAM;AAAA,MACzB,UAAU,MAAM,CAAC,GAAG,IAAI,CAAC;AAAA,MACzB,UAAU,UAAU,CAAC,OAAO,IAAI,CAAC;AAAA,MACjC,OAAO,oBAAoB,SAAS;AAAA,MACpC,QAAQ,oBAAoB,UAAU;AAAA,IACxC,CAAC;AAAA,EACH;AACA,QAAM,aAAa,CAAC,OAA2B,QAAgB,MAAiB,QAAwB;AACtG,eAAW,SAAS,SAAS,IAAI,MAAM,GAAG,GAAG;AAC3C,YAAM,CAAC,KAAK,UAAU,IAAI,KAAK,KAAK,EAAE,MAAM,KAAK;AACjD,UAAI,CAAC,IAAK;AACV,YAAM,QAAQ,YAAY,MAAM,UAAU,IAAI,CAAC;AAC/C,UAAI,KAAK,QAAQ,MAAM,KAAK,MAAM,QAAQ,EAAE,OAAO,OAAO,KAAK,GAAG,QAAQ,EAAE,IAAI,IAAI;AAAA,IACtF;AAAA,EACF;AACA,QAAM,OAAO,CAACA,WAAwB,SAAS,OAAO;AACpD,UAAM,IAAIA,UAAS;AACnB,eAAW,SAASA,UAAS,QAAQ;AACnC,YAAM,QAAQ,MAAM;AACpB,YAAM,MAAM,YAAY,MAAM,KAAK,GAAG;AACtC,YAAM,aAAa,MAAM,SAAS,MAAM,SAAS,EAAE,OAAO,MAAM,OAAO,QAAQ,MAAM,OAAO,IAAI;AAChG,iBAAW,QAAQ,CAAC,OAAO,YAAY,iBAAiB,iBAAiB,WAAW,mBAAmB,eAAe,gBAAgB,WAAW,GAAG;AAClJ,cAAM,QAAQ,MAAM,IAAI;AACxB,YAAI,OAAO,UAAU,IAAI,GAAG,MAAM,IAAI,SAAS,KAAK,MAAM,UAAU;AAAA,MACtE;AACA,iBAAW,MAAM,QAAQ,gBAAgB,MAAM,IAAI,SAAS,GAAG;AAC/D,iBAAW,MAAM,aAAa,GAAG,qBAAqB,MAAM,IAAI,SAAS,GAAG;AAAA,IAC9E;AACA,MAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,YAAY;AACpC,YAAM,OAAO,EAAE,OAAO;AACtB,YAAM,WAAW,KAAK,KAAK,MAAM,KAAK;AACtC,YAAM,OAAkB,SAAS,WAAW,QAAQ,IAAI,UAAU,SAAS,WAAW,QAAQ,IAAI,UAAU;AAC5G,UAAI,KAAK,KAAK,KAAK,GAAG,gBAAgB,MAAM,IAAI,IAAI;AACpD,iBAAW,KAAK,KAAK,QAAQ,GAAG,uBAAuB,MAAM,IAAI,IAAI;AAAA,IACvE,CAAC;AACD,MAAE,aAAa,EAAE,KAAK,CAAC,QAAQ,YAAY;AACzC,YAAM,OAAO,EAAE,OAAO;AACtB,YAAM,OAAO,QAAQ,QAAQ,YAAY;AACzC,UAAI,KAAK,KAAK,KAAK,GAAG,UAAU,IAAI,GAAG,MAAM,IAAI,IAAI;AACrD,UAAI,SAAS,QAAS,KAAI,KAAK,KAAK,QAAQ,GAAG,sBAAsB,MAAM,IAAI,OAAO;AAAA,IACxF,CAAC;AACD,MAAE,MAAM,EAAE,KAAK,CAAC,QAAQ,YAAY;AAClC,YAAM,OAAO,EAAE,OAAO;AACtB,YAAM,QAAQ,KAAK,KAAK,UAAU,KAAK,KAAK,KAAK,MAAM,KAAK,IAAI,YAAY;AAC5E,UAAI,SAAS,cAAc,SAAS,kBAAkB,SAAS,mBAAmB,SAAS,qBAAqB;AAC9G,YAAI,KAAK,KAAK,SAAS,GAAG,UAAU,IAAI,GAAG,MAAM,IAAI,OAAO;AAAA,MAC9D;AAAA,IACF,CAAC;AACD,MAAE,YAAY,EAAE,KAAK,CAAC,QAAQ,YAAY;AACxC,YAAM,OAAO,EAAE,OAAO;AACtB,YAAM,OAAO,KAAK,KAAK,KAAK,KAAK,IAAI,YAAY,EAAE,MAAM,KAAK;AAC9D,UAAI,IAAI,KAAK,WAAS,UAAU,UAAU,UAAU,kBAAkB,GAAG;AACvE,YAAI,KAAK,KAAK,MAAM,GAAG,cAAc,MAAM,IAAI,OAAO;AAAA,MACxD;AAAA,IACF,CAAC;AACD,MAAE,WAAW,EAAE,KAAK,CAAC,QAAQ,YAAY;AACvC,YAAM,OAAO,EAAE,OAAO;AACtB,UAAI,KAAK,KAAK,MAAM,KAAK,KAAK,KAAK,YAAY,GAAG,mBAAmB,MAAM,IAAI,OAAO;AAAA,IACxF,CAAC;AACD,MAAE,SAAS,EAAE,IAAI,KAAK,EAAE,KAAK,CAAC,QAAQ,YAAY;AAChD,YAAM,OAAO,EAAE,OAAO;AACtB,YAAM,OAAO,KAAK,KAAK,MAAM;AAC7B,UAAI,CAAC,KAAM;AACX,UACE,wEAAwE,KAAK,GAAG,KAAK,KAAK,OAAO,KAAK,EAAE,IAAI,KAAK,KAAK,8BAA8B,KAAK,EAAE,EAAE,KAC1J,+CAA+C,KAAK,IAAI,GAC3D;AACA,cAAM,MAAM,YAAY,KAAK,KAAK,KAAK,EAAE,MAAM,EAAE,KAAK,KAAK,GAAG,GAAG;AACjE,YAAI,MAAM,yBAAyB,MAAM,IAAI,SAAS,GAAG;AAAA,MAC3D;AAAA,IACF,CAAC;AACD,UAAM,aAAa,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,aAAW,EAAE,OAAO,EAAE,KAAK,CAAC;AACxE,MAAE,SAAS,EAAE,KAAK,CAAC,QAAQ,YAAY;AAAE,iBAAW,KAAK,EAAE,OAAO,EAAE,KAAK,OAAO,KAAK,EAAE;AAAA,IAAE,CAAC;AAC1F,eAAW,SAAS,WAAW,KAAK,IAAI,EAAE,SAAS,0IAA0I,GAAG;AAC9L,UAAI,MAAM,CAAC,GAAG,iBAAiB,MAAM,IAAI,OAAO;AAAA,IAClD;AACA,eAAW,SAASA,UAAS,WAAW,SAAS,sGAAsG,GAAG;AACxJ,UAAI,MAAM,CAAC,GAAG,kBAAkB,MAAM,EAAE;AAAA,IAC1C;AAAA,EACF;AAEA,QAAM,WAAW,OAAO,SAAS,WAAW,iBAAiB,IAAI,IAAI;AACrE,OAAK,QAAQ;AACb,WAAS,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,YAAY;AAC7C,UAAM,SAAS,SAAS,EAAE,OAAO,EAAE,KAAK;AACxC,UAAM,YAAY,OACf,QAAQ,eAAe,GAAG,EAAE,QAAQ,eAAe,GAAG,EAAE,QAAQ,aAAa,GAAG,EAChF,QAAQ,WAAW,WAAS,MAAM,MAAM,CAAC,CAAC,EAAE,QAAQ,SAAS,GAAG;AACnE,QAAI,cAAc,UAAU,iCAAiC,KAAK,SAAS,GAAG;AAC5E,WAAK,iBAAiB,SAAS,GAAG,iBAAiB;AAAA,IACrD;AAAA,EACF,CAAC;AACD,SAAO,CAAC,GAAG,MAAM,OAAO,CAAC;AAC3B;AAEO,SAAS,oBACd,cACA,UACA,YAAY,KAC8C;AAC1D,QAAM,QAAQ,oBAAI,IAAsC;AACxD,aAAW,SAAS,aAAc,gBAAe,OAAO,KAAK;AAC7D,aAAW,SAAS,UAAU,UAAU,CAAC,EAAG,gBAAe,OAAO,EAAE,GAAG,OAAO,MAAM,MAAM,KAAK,CAAC;AAEhG,QAAM,WAAW,oBAAI,IAAwC;AAC7D,aAAW,SAAS,MAAM,OAAO,GAAG;AAClC,UAAM,MAAM,eAAe,MAAM,GAAG;AACpC,UAAM,SAAS,SAAS,IAAI,GAAG,KAAK,CAAC;AACrC,WAAO,KAAK,KAAK;AACjB,aAAS,IAAI,KAAK,MAAM;AAAA,EAC1B;AACA,SAAO,CAAC,GAAG,SAAS,OAAO,CAAC,EACzB,IAAI,YAAU;AACb,UAAM,UAAU,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,eAAe,CAAC,IAAI,eAAe,CAAC,KAAK,EAAE,IAAI,cAAc,EAAE,GAAG,CAAC;AAC9G,UAAM,YAAY,EAAE,GAAG,QAAQ,CAAC,EAAE;AAClC,cAAU,mBAAmB,CAAC,GAAG,IAAI,IAAI,OAAO,QAAQ,WAAS,MAAM,gBAAgB,CAAC,CAAC;AACzF,cAAU,WAAW,CAAC,GAAG,IAAI,IAAI,OAAO,QAAQ,WAAS,MAAM,QAAQ,CAAC,CAAC;AACzE,cAAU,WAAW,CAAC,GAAG,IAAI,IAAI,OAAO,QAAQ,WAAS,MAAM,QAAQ,CAAC,CAAC;AACzE,WAAO,EAAE,GAAG,WAAW,UAAU,QAAQ,MAAM,CAAC,EAAE,IAAI,WAAS,MAAM,GAAG,EAAE;AAAA,EAC5E,CAAC,EACA,KAAK,CAAC,GAAG,MAAM,eAAe,CAAC,IAAI,eAAe,CAAC,KAAK,EAAE,IAAI,cAAc,EAAE,GAAG,CAAC,EAClF,MAAM,GAAG,KAAK,IAAI,GAAG,SAAS,CAAC;AACpC;AAMA,eAAsB,cACpB,KACA,SACA,UACA,UAII,CAAC,GACuE;AAC5E,QAAM,WAAW,KAAK,IAAI,GAAG,KAAK,IAAI,QAAQ,YAAY,KAAK,OAAO,MAAM,MAAM,OAAO,IAAI,CAAC;AAC9F,MAAI,SAAS;AACb,MAAI,MAAuB;AAC3B,WAAS,YAAY,GAAG,aAAa,GAAG,aAAa;AACnD,UAAM,UAAU,MAAM,sBAAsB,QAAQ,EAAE,OAAO,YAAY,CAAC;AAC1E,QAAI,QAAQ,SAAS,CAAC,QAAQ,OAAQ,OAAM,IAAI,MAAM,QAAQ,SAAS,wBAAwB;AAC/F,UAAM,MAAM,MAAM,QAAQ,OAAO,MAAM;AAAA,MACrC,QAAQ,YAAY,QAAQ,IAAM;AAAA,MAClC,UAAU;AAAA,IACZ,CAAC;AACD,QAAI,IAAI,UAAU,OAAO,IAAI,SAAS,KAAK;AACzC,YAAM,WAAW,IAAI,QAAQ,IAAI,UAAU;AAC3C,UAAI,CAAC,SAAU,OAAM,IAAI,MAAM,QAAQ,IAAI,MAAM,oCAAoC;AACrF,eAAS,IAAI,IAAI,UAAU,QAAQ,OAAO,IAAI,EAAE;AAChD,YAAM;AACN;AAAA,IACF;AACA;AAAA,EACF;AACA,MAAI,CAAC,IAAK,OAAM,IAAI,MAAM,wCAAwC;AAClE,MAAI,CAAC,IAAI,GAAI,OAAM,IAAI,MAAM,QAAQ,IAAI,MAAM,EAAE;AACjD,MAAI,CAAC,IAAI,KAAM,OAAM,IAAI,MAAM,qBAAqB;AAEpD,QAAM,WAAW,IAAI,QAAQ,IAAI,cAAc,GAAG,MAAM,GAAG,EAAE,CAAC,EAAE,KAAK,KAAK;AAC1E,MAAI,QAAQ,iBAAiB,CAAC,YAAY,aAAa,QAAQ,MAAM,QAAQ,eAAe;AAC1F,UAAM,IAAI,MAAM,YAAY,QAAQ,YAAY,yBAAyB,YAAY,iBAAiB,EAAE;AAAA,EAC1G;AACA,QAAM,gBAAgB,OAAO,IAAI,QAAQ,IAAI,gBAAgB,KAAK,CAAC;AACnE,MAAI,OAAO,SAAS,aAAa,KAAK,gBAAgB,UAAU;AAC9D,UAAM,IAAI,MAAM,uBAAuB,QAAQ,aAAa;AAAA,EAC9D;AAEA,MAAI,OAAO,KAAK,SAAS,QAAQ;AACjC,MAAI,YAAY,CAAC,QAAQ,QAAQ,GAAG;AAClC,UAAM,UAAkC;AAAA,MACtC,cAAc;AAAA,MAAQ,aAAa;AAAA,MAAQ,cAAc;AAAA,MACzD,aAAa;AAAA,MAAQ,iBAAiB;AAAA,MAAQ,cAAc;AAAA,MAC5D,aAAa;AAAA,MAAQ,cAAc;AAAA,MACnC,cAAc;AAAA,MAAQ,aAAa;AAAA,MAAQ,aAAa;AAAA,IAC1D;AACA,UAAM,MAAM,QAAQ,QAAQ;AAC5B,QAAI,IAAK,QAAO,OAAO;AAAA,EACzB;AAEA,QAAM,SAAS,kBAAkB,IAAI;AACrC,QAAM,IAAI,QAAc,CAAC,SAAS,WAAW;AAC3C,WAAO,KAAK,QAAQ,MAAM,QAAQ,CAAC;AACnC,WAAO,KAAK,SAAS,MAAM;AAAA,EAC7B,CAAC;AACD,MAAI,gBAAgB;AACpB,QAAM,UAAU,IAAI,UAAU;AAAA,IAC5B,UAAU,OAAe,WAAW,UAAU;AAC5C,YAAM,QAAQ,MAAM;AACpB,UAAI,gBAAgB,QAAQ,UAAU;AACpC,iBAAS,IAAI,MAAM,uBAAuB,QAAQ,aAAa,CAAC;AAChE;AAAA,MACF;AACA,UAAI,QAAQ,gBAAgB,CAAC,QAAQ,aAAa,KAAK,GAAG;AACxD,iBAAS,IAAI,MAAM,wCAAwC,CAAC;AAC5D;AAAA,MACF;AACA,uBAAiB;AACjB,eAAS,MAAM,KAAK;AAAA,IACtB;AAAA,EACF,CAAC;AACD,MAAI;AACF,UAAM,SAAS,SAAS,QAAQ,IAAI,IAA2C,GAAG,SAAS,MAAM;AAAA,EACnG,SAAS,OAAO;AACd,WAAO,QAAQ;AACf,WAAO,MAAM,EAAE,OAAO,KAAK,CAAC;AAC5B,UAAM;AAAA,EACR;AAEA,QAAM,EAAE,SAAS,IAAI,MAAM,OAAO,IAAS;AAC3C,QAAM,YAAY,SAAS,IAAI,EAAE;AAEjC,SAAO,EAAE,WAAW,MAAM,WAAW,SAAS;AAChD;AAEA,eAAsB,iBACpB,MACA,SACA,UAA+B,CAAC,GACR;AACxB,QAAM,QAAW,QAAQ,SAAS,CAAC,SAAS,SAAS,OAAO;AAC5D,QAAM,WAAW,IAAI,IAAI,KAAK;AAC9B,QAAM,eAAe,oBAAoB,MAAM,OAAO;AACtD,QAAM,YAAY,KAAK,IAAI,GAAG,KAAK,IAAI,QAAQ,aAAa,KAAK,GAAG,CAAC;AACrE,QAAM,gBAAgB,oBAAoB,cAAc,QAAQ,UAAU,GAAK;AAC/E,QAAM,aAAa,cAAc,MAAM,GAAG,SAAS;AACnD,QAAM,cAAa,oBAAI,IAAI;AAAA,IACzB,GAAG,aAAa,IAAI,WAAS,MAAM,GAAG;AAAA,IACtC,IAAI,QAAQ,UAAU,UAAU,CAAC,GAAG,IAAI,WAAS,MAAM,GAAG;AAAA,EAC5D,CAAC,GAAE;AACH,QAAM,OAAO,WAAW,QAAQ,WAAS;AACvC,UAAM,OAAO,MAAM,QAAQ,YAAY,MAAM,GAAG;AAChD,WAAO,QAAQ,SAAS,IAAI,IAAI,IAAI,CAAC,EAAE,GAAG,OAAO,KAAK,CAAC,IAAI,CAAC;AAAA,EAC9D,CAAC;AACD,QAAM,gBAAgB,aAAa,KAAK;AACxC,QAAM,wBAAwB,cAAc,SAAS;AACrD,QAAM,YAAY,QAAQ,UAAU,cAAc,QAAQ,CAAC;AAC3D,QAAM,aAAa,wBAAwB,gBAAgB,QAAQ,UAAU,cAAc;AAC3F,QAAM,WAAW,wBACb,CAAC,yBAAyB,cAAc,MAAM,4CAA4C,SAAS,yBAAyB,IAC5H,QAAQ,WACR,QAAQ,SAAS,YAAY,CAAC,IAAI,CAAC,yCAAyC,QAAQ,SAAS,UAAU,oCAAoC,IAC3I,CAAC,+FAA+F;AAEpG,QAAM,eAA4D;AAAA,IAChE;AAAA,IACA,aAAa,aAAa;AAAA,IAC1B,eAAe,QAAQ,UAAU,OAAO,UAAU;AAAA,IAClD;AAAA,IACA;AAAA,IACA,cAAc,YAAY,aAAsB;AAAA,IAChD;AAAA,IACA;AAAA,IACA,cAAc,QAAQ,UAAU,gBAAgB;AAAA,IAChD;AAAA,IACA,UAAU;AAAA,EACZ;AAEA,MAAI,QAAQ,cAAc,MAAM;AAC9B,WAAO;AAAA,MACL,GAAG;AAAA,MACH,WAAW;AAAA,MACX,QAAQ,KAAK,IAAI,CAAC,OAAO,OAAO;AAAA,QAC9B,GAAG;AAAA,QACH,UAAU;AAAA,QAAM,UAAU,aAAa,MAAM,KAAK,CAAC;AAAA,QAAG,WAAW;AAAA,QAAM,WAAW;AAAA,QAClF,QAAQ;AAAA,QAAM,gBAAgB;AAAA,QAA0B,eAAe;AAAA,QAAM,eAAe;AAAA,MAC9F,EAAE;AAAA,IACJ;AAAA,EACF;AAEA,QAAM,UAAW,MAAM;AAAE,QAAI;AAAE,aAAO,IAAI,IAAI,OAAO,EAAE,SAAS,QAAQ,UAAU,EAAE;AAAA,IAAE,QAAQ;AAAE,aAAO;AAAA,IAAU;AAAA,EAAE,GAAG;AACtH,QAAM,SAAU,oBAAI,KAAK,GAAE,YAAY,EAAE,QAAQ,SAAS,GAAG,EAAE,MAAM,GAAG,EAAE;AAC1E,QAAM,SAAU,QAAQ,aAAa,KAAK,QAAQ,GAAG,aAAa,eAAe,SAAS,GAAG,KAAK,IAAI,MAAM,EAAE;AAC9G,YAAU,QAAQ,EAAE,WAAW,KAAK,CAAC;AAErC,QAAM,SAAuB,CAAC;AAE9B,QAAM,QAAQ;AAAA,IACZ,KAAK,IAAI,OAAO,OAAO,MAAM;AAC3B,YAAM,WAAW,aAAa,MAAM,KAAK,CAAC;AAC1C,UAAI;AACF,cAAM,EAAE,WAAW,WAAW,SAAS,IAAI,MAAM,cAAc,MAAM,KAAK,QAAQ,QAAQ;AAC1F,cAAM,eAAe,WAAY,aAAa,QAAQ,KAAK,MAAM,OAAQ,MAAM;AAC/E,eAAO,KAAK;AAAA,UACV,GAAG;AAAA,UAAO,MAAM;AAAA,UAAc;AAAA,UAAU,UAAU,SAAS,SAAS;AAAA,UAAG;AAAA,UAAW;AAAA,UAClF,QAAQ;AAAA,UAAM,gBAAgB;AAAA,UAAc,eAAe;AAAA,UAAM,eAAe;AAAA,QAClF,CAAC;AAAA,MACH,SAAS,OAAO;AACd,eAAO,KAAK;AAAA,UACV,GAAG;AAAA,UAAO,UAAU;AAAA,UAAM;AAAA,UAAU,WAAW;AAAA,UAAM,WAAW;AAAA,UAAM,QAAQ;AAAA,UAC9E,gBAAgB;AAAA,UAAU,eAAe,iBAAiB,QAAQ,MAAM,UAAU;AAAA,UAAyB,eAAe;AAAA,QAC5H,CAAC;AAAA,MACH;AAAA,IACF,CAAC;AAAA,EACH;AAEA,SAAO,KAAK,CAAC,GAAG,MAAM;AACpB,QAAI,EAAE,aAAa,CAAC,EAAE,UAAW,QAAO;AACxC,QAAI,CAAC,EAAE,aAAa,EAAE,UAAW,QAAO;AACxC,WAAO,EAAE,IAAI,cAAc,EAAE,GAAG;AAAA,EAClC,CAAC;AAED,SAAO,EAAE,GAAG,cAAc,WAAW,QAAQ,OAAO;AACtD;;;AC7dA,IAAM,oBAAoB,oBAAI,IAAI;AAAA,EAChC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAED,SAAS,UAAU,KAAkC;AACnD,SAAO,eAAe,QAAS,IAAkC,OAAO;AAC1E;AAEA,SAAS,aAAa,KAAsB;AAC1C,SAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AACxD;AAEA,SAAS,8BAA8B,OAA0C;AAC/E,QAAM,QAAQ,OAAO,MAAM,sBAAsB;AACjD,SAAO,QAAQ,OAAO,MAAM,CAAC,CAAC,IAAI;AACpC;AAEA,SAAS,iBAAiB,QAAqC;AAC7D,MAAI,WAAW,IAAK,QAAO;AAC3B,MAAI,WAAW,IAAK,QAAO;AAC3B,MAAI,WAAW,IAAK,QAAO;AAC3B,MAAI,UAAU,OAAO,UAAU,IAAK,QAAO;AAC3C,SAAO;AACT;AAEO,SAAS,0BAA0B,KAAiC;AACzE,QAAM,UAAU,aAAa,GAAG;AAChC,QAAM,OAAO,UAAU,GAAG;AAE1B,MAAI,yBAAyB,KAAK,OAAO,GAAG;AAC1C,WAAO,EAAE,WAAW,sBAAsB,YAAY,MAAM,WAAW,KAAK;AAAA,EAC9E;AAEA,QAAM,aAAa,8BAA8B,IAAI,KAAK,8BAA8B,QAAQ,MAAM,qBAAqB,IAAI,CAAC,CAAC;AACjI,MAAI,cAAc,MAAM;AACtB,WAAO,EAAE,WAAW,iBAAiB,UAAU,GAAG,YAAY,WAAW,eAAe,OAAO,cAAc,IAAI;AAAA,EACnH;AAEA,MAAI,SAAS,kCAAkC,gCAAgC,KAAK,OAAO,GAAG;AAC5F,WAAO,EAAE,WAAW,wBAAwB,YAAY,MAAM,WAAW,KAAK;AAAA,EAChF;AAEA,MAAI,SAAS,gCAAgC,0BAA0B,KAAK,OAAO,GAAG;AACpF,WAAO,EAAE,WAAW,kBAAkB,YAAY,MAAM,WAAW,MAAM;AAAA,EAC3E;AAEA,MAAI,8GAA8G,KAAK,OAAO,GAAG;AAC/H,WAAO,EAAE,WAAW,+BAA+B,YAAY,MAAM,WAAW,KAAK;AAAA,EACvF;AAEA,MAAI,kBAAkB,IAAI,QAAQ,EAAE,KAAK,kCAAkC,KAAK,OAAO,GAAG;AACxF,WAAO,EAAE,WAAW,oBAAoB,YAAY,MAAM,WAAW,KAAK;AAAA,EAC5E;AAEA,SAAO,EAAE,WAAW,qBAAqB,YAAY,MAAM,WAAW,MAAM;AAC9E;AAEO,SAAS,6BAA6B,MAA2B,YAAmC;AACzG,UAAQ,MAAM;AAAA,IACZ,KAAK;AACH,aAAO,qCAAqC,cAAc,GAAG;AAAA,IAC/D,KAAK;AACH,aAAO,sCAAsC,cAAc,GAAG;AAAA,IAChE,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO,iDAAiD,UAAU;AAAA,IACpE,KAAK;AACH,aAAO,wCAAwC,UAAU;AAAA,IAC3D,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,EACX;AACF;AAEO,SAAS,0BAA0B,SAA4B,UAAkC,CAAC,GAA4B;AACnI,QAAM,UAAU,6BAA6B,QAAQ,WAAW,QAAQ,UAAU;AAClF,QAAM,WAAW,yBAAyB;AAAA,IACxC,WAAW,QAAQ;AAAA,IACnB,WAAW,QAAQ;AAAA,IACnB,cAAc,QAAQ;AAAA,IACtB,SAAS;AAAA,MACP,GAAI,QAAQ,WAAW,CAAC;AAAA,MACxB,GAAI,QAAQ,cAAc,OAAO,EAAE,aAAa,QAAQ,WAAW,IAAI,CAAC;AAAA,IAC1E;AAAA,EACF,CAAC;AACD,SAAO,EAAE,GAAG,UAAU,SAAS,OAAO,QAAQ;AAChD;AAEO,SAAS,qBAAqB,SAA+D;AAClG,MAAI,QAAQ,eAAe,IAAK,QAAO;AACvC,MAAI,QAAQ,eAAe,IAAK,QAAO;AACvC,MAAI,QAAQ,eAAe,IAAK,QAAO;AACvC,MAAI,QAAQ,cAAc,QAAQ,QAAQ,cAAc,OAAO,QAAQ,cAAc,IAAK,QAAO;AACjG,MAAI,QAAQ,cAAc,KAAM,QAAO;AACvC,MAAI,QAAQ,cAAc,qBAAsB,QAAO;AACvD,SAAO;AACT;AAEA,IAAM,0BAA0B;AAEzB,SAAS,2BACd,aACA,eAC8D;AAC9D,MAAI,eAAe,KAAM,QAAO,EAAE,aAAa,MAAM,eAAe,KAAK;AACzE,MAAI,gBAAgB,wBAAyB,QAAO,EAAE,aAAa,eAAe,iBAAiB,KAAK;AACxG,QAAM,MAAM,IAAI,MAAM,iBAAiB,WAAW;AAClD,MAAI,OAAO;AACX,QAAM,UAAU,0BAA0B,GAAG;AAC7C,SAAO,EAAE,aAAa,QAAQ,WAAW,eAAe,6BAA6B,QAAQ,WAAW,QAAQ,UAAU,EAAE;AAC9H;;;AC9JA,SAAS,YAAoC;AAGtC,IAAM,yBAAyB;AAEtC,IAAI,aAAqD;AACzD,IAAI,oBAAoB;AAEjB,SAAS,oBAA4B;AAC1C,UAAQ,QAAQ,IAAI,oBAAoB,iCAAiC,KAAK;AAChF;AAEO,SAAS,kBAA0B;AACxC,SAAO,QAAQ,QAAQ,IAAI,kBAAkB,QAAQ,KAAK,CAAC;AAC7D;AAEO,SAAS,kCAA2C;AACzD,SAAO,QAAQ,QAAQ,IAAI,cAAc,KAAK,KAAK,QAAQ,IAAI,qBAAqB,KAAK,CAAC;AAC5F;AAEA,SAAS,YAA6C;AACpD,MAAI,WAAY,QAAO;AACvB,QAAM,MAAM,QAAQ,IAAI,qBAAqB,KAAK;AAClD,MAAI,CAAC,IAAK,OAAM,IAAI,MAAM,oFAAoF;AAC9G,eAAa,KAAK,GAAG;AACrB,SAAO;AACT;AAOA,eAAsB,4BAA2C;AAC/D,MAAI,kBAAmB;AACvB,QAAM,YAAY,gBAAgB;AAClC,QAAM,UAAU,EAAE,MAAM,uCAAuC;AAC/D,QAAM,UAAU,EAAE,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,6BAMG,SAAS;AAAA;AAAA;AAAA;AAAA,GAInC;AACD,QAAM,UAAU,EAAE,MAAM,6FAA6F;AACrH,sBAAoB;AACtB;AAEA,eAAsB,kBAAkB,OAAsC;AAC5E,QAAM,SAAS,QAAQ,IAAI,cAAc,KAAK;AAC9C,MAAI,CAAC,OAAQ,OAAM,IAAI,MAAM,gEAAgE;AAC7F,MAAI,CAAC,MAAM,OAAQ,QAAO,CAAC;AAC3B,QAAM,WAAW,MAAM,MAAM,qCAAqC;AAAA,IAChE,QAAQ;AAAA,IACR,SAAS,EAAE,eAAe,UAAU,MAAM,IAAI,gBAAgB,mBAAmB;AAAA,IACjF,MAAM,KAAK,UAAU;AAAA,MACnB,OAAO,kBAAkB;AAAA,MACzB,YAAY,gBAAgB;AAAA,MAC5B,OAAO,MAAM,IAAI,WAAS,EAAE,MAAM,KAAK,MAAM,GAAG,GAAK,EAAE,EAAE;AAAA,IAC3D,CAAC;AAAA,IACD,QAAQ,YAAY,QAAQ,GAAM;AAAA,EACpC,CAAC;AACD,MAAI,CAAC,SAAS,IAAI;AAChB,UAAM,IAAI,MAAM,2CAA2C,SAAS,MAAM,MAAM,MAAM,SAAS,KAAK,GAAG,MAAM,GAAG,GAAG,CAAC,EAAE;AAAA,EACxH;AACA,QAAM,UAAU,MAAM,SAAS,KAAK;AACpC,QAAM,WAAW,QAAQ,QAAQ,CAAC,GAAG,IAAI,UAAQ,KAAK,SAAS;AAC/D,MAAI,QAAQ,WAAW,MAAM,QAAQ;AACnC,UAAM,IAAI,MAAM,iBAAiB,QAAQ,MAAM,mBAAmB,MAAM,MAAM,UAAU;AAAA,EAC1F;AACA,SAAO;AACT;AAEA,eAAsB,4BAA4B,QAAQ,IAKvD;AACD,MAAI,CAAC,gCAAgC,EAAG,QAAO,EAAE,SAAS,GAAG,UAAU,GAAG,QAAQ,GAAG,WAAW,EAAE;AAClG,QAAM,0BAA0B;AAChC,QAAM,UAAU,KAAK,IAAI,GAAG,KAAK,IAAI,KAAK,KAAK,MAAM,KAAK,CAAC,CAAC;AAC5D,QAAM,SAAS,MAAM,MAAM,EAAE,QAAQ;AAAA,IACnC,KAAK;AAAA;AAAA,IAEL,MAAM,CAAC,OAAO;AAAA,EAChB,CAAC;AACD,QAAM,OAAO,OAAO;AACpB,MAAI,CAAC,KAAK,OAAQ,QAAO,EAAE,SAAS,GAAG,UAAU,GAAG,QAAQ,GAAG,WAAW,MAAM,2BAA2B,EAAE;AAE7G,MAAI,WAAW;AACf,MAAI,SAAS;AACb,QAAM,QAAQ,kBAAkB;AAChC,MAAI;AACF,UAAM,UAAU,MAAM,kBAAkB,KAAK,IAAI,SAAO,GAAG,IAAI,KAAK;AAAA;AAAA,EAAO,IAAI,IAAI,EAAE,CAAC;AACtF,eAAW,CAAC,OAAO,GAAG,KAAK,KAAK,QAAQ,GAAG;AACzC,YAAM,UAAU,IAAI,QAAQ,KAAK,EAAE,KAAK,GAAG,CAAC;AAC5C,YAAM,UAAU,EAAE;AAAA,QAChB;AAAA;AAAA;AAAA;AAAA,QAIA,CAAC,IAAI,IAAI,IAAI,WAAW,IAAI,eAAe,IAAI,OAAO,SAAS,KAAK;AAAA,MACtE;AACA,YAAM,MAAM,EAAE,QAAQ;AAAA,QACpB,KAAK;AAAA;AAAA,QAEL,MAAM,CAAC,wBAAwB,OAAO,IAAI,KAAI,oBAAI,KAAK,GAAE,YAAY,GAAG,IAAI,EAAE;AAAA,MAChF,CAAC;AACD,kBAAY;AAAA,IACd;AAAA,EACF,SAAS,OAAO;AACd,aAAS,KAAK,SAAS;AACvB,UAAM,WAAW,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,GAAG,MAAM,GAAG,GAAG;AACrF,eAAW,OAAO,KAAK,MAAM,QAAQ,GAAG;AACtC,YAAM,MAAM,EAAE,QAAQ;AAAA,QACpB,KAAK;AAAA,QACL,MAAM,CAAC,SAAS,IAAI,EAAE;AAAA,MACxB,CAAC,EAAE,MAAM,MAAM,MAAS;AAAA,IAC1B;AAAA,EACF;AACA,SAAO,EAAE,SAAS,KAAK,QAAQ,UAAU,QAAQ,WAAW,MAAM,2BAA2B,EAAE;AACjG;AAEA,eAAsB,6BAA8C;AAClE,QAAM,SAAS,MAAM,MAAM,EAAE,QAAQ,kGAAkG;AACvI,SAAO,OAAQ,OAAO,KAAK,CAAC,GAAgC,KAAK,CAAC;AACpE;AAEA,eAAsB,4BAA4B,OAAe,QAAQ,IAAkC;AACzG,QAAM,SAAS,oBAAI,IAAoB;AACvC,MAAI,CAAC,gCAAgC,KAAK,CAAC,MAAM,KAAK,EAAG,QAAO;AAChE,QAAM,0BAA0B;AAChC,QAAM,CAAC,MAAM,IAAI,MAAM,kBAAkB,CAAC,KAAK,CAAC;AAChD,MAAI,CAAC,OAAQ,QAAO;AACpB,QAAM,OAAO,MAAM,UAAU,EAAE;AAAA,IAC7B;AAAA;AAAA,IAEA,CAAC,IAAI,OAAO,KAAK,GAAG,CAAC,KAAK,KAAK,IAAI,GAAG,KAAK,IAAI,KAAK,KAAK,CAAC,CAAC;AAAA,EAC7D;AACA,aAAW,OAAO,KAAM,QAAO,IAAI,OAAO,IAAI,SAAS,GAAG,OAAO,IAAI,KAAK,CAAC;AAC3E,SAAO;AACT;","names":["document"]}
@@ -50,7 +50,7 @@ function renderInstallTerminal(options) {
50
50
  "1/1 install surfaces ready",
51
51
  colorize("Newest: any approved connection read can become an indexed Memory snapshot in one call. OAuth stays tenant-isolated and provider content is redacted and marked untrusted.", "lime", color),
52
52
  "",
53
- `${colorize("Tools", "cyan", color)} ${colorize("(258 MCP tools)", "muted", color)}`,
53
+ `${colorize("Tools", "cyan", color)} ${colorize("(259 MCP tools)", "muted", color)}`,
54
54
  toolRow("search", ["harvest_paa", "search_serp", "maps_search", "maps_place_intel"], color),
55
55
  toolRow("extract", ["extract_url", "map_site_urls", "extract_site", "audit_site", "directory_workflow"], color),
56
56
  toolRow("build", ["create_editorial_reading_room", "rank_tracker_workflow", "portable HTML"], color),
@@ -106,4 +106,4 @@ function renderInstallTerminal(options) {
106
106
  export {
107
107
  renderInstallTerminal
108
108
  };
109
- //# sourceMappingURL=chunk-3NXLGTC6.js.map
109
+ //# sourceMappingURL=chunk-DCWXVAQT.js.map
@@ -1 +1 @@
1
- {"version":3,"sources":["../src/install-terminal.ts"],"sourcesContent":["export interface InstallTerminalOptions {\n color?: boolean\n version: string\n apiKeyConfigured?: boolean\n}\n\ntype Tone = 'cyan' | 'lime' | 'amber' | 'red' | 'muted' | 'bold'\n\nconst CODES: Record<Tone | 'reset', string> = {\n reset: '\\x1b[0m',\n cyan: '\\x1b[36m',\n lime: '\\x1b[32m',\n amber: '\\x1b[33m',\n red: '\\x1b[31m',\n muted: '\\x1b[90m',\n bold: '\\x1b[1m',\n}\n\nfunction colorize(value: string, tone: Tone, enabled: boolean): string {\n if (!enabled) return value\n return `${CODES[tone]}${value}${CODES.reset}`\n}\n\nfunction toolRow(label: string, tools: string[], enabled: boolean): string {\n const padded = label.padEnd(9, ' ')\n return ` ${colorize(padded, 'muted', enabled)} ${tools.join(colorize(' . ', 'muted', enabled))}`\n}\n\nexport function renderInstallTerminal(options: InstallTerminalOptions): string {\n const color = options.color ?? true\n const apiKeyValue = options.apiKeyConfigured ? '$MCP_SCRAPER_API_KEY' : 'sk_live_your_key'\n const ascii = String.raw`\n __ __ ____ ____ \n| \\/ |/ ___| _ \\\n| |\\/| | | | |_) |\n| | | | |___| __/\n|_| |_|\\____|_|\n\n ____ ____ ____ _ ____ _____ ____\n/ ___| / ___| _ \\ / \\ | _ \\| ____| _ \\\n\\___ \\| | | |_) | / _ \\ | |_) | _| | |_) |\n ___) | |___| _ < / ___ \\| __/| |___| _ <\n|____/ \\____|_| \\_\\/_/ \\_\\_| |_____|_| \\_\\\n`\n\n const claudeCommand = [\n `MCP_SCRAPER_API_KEY=${apiKeyValue} npx -y -p mcp-scraper@latest \\\\`,\n ' mcp-scraper-cli agent install claude --apply',\n ].join('\\n')\n\n const codexConfig = [\n '[mcp_servers.mcp-scraper]',\n 'command = \"npx\"',\n 'args = [\"-y\", \"-p\", \"mcp-scraper@latest\", \"mcp-scraper\"]',\n `env = { MCP_SCRAPER_API_KEY = \"${apiKeyValue}\" }`,\n ].join('\\n')\n\n return [\n colorize(`mcp-scraper v${options.version}`, 'bold', color),\n colorize('> mcp-scraper-install', 'muted', color),\n colorize(ascii, 'amber', color),\n `${colorize('MCP Scraper Agent', 'cyan', color)} . v${options.version} . mcpscraper.dev`,\n '1/1 install surfaces ready',\n colorize('Newest: any approved connection read can become an indexed Memory snapshot in one call. OAuth stays tenant-isolated and provider content is redacted and marked untrusted.', 'lime', color),\n '',\n `${colorize('Tools', 'cyan', color)} ${colorize('(258 MCP tools)', 'muted', color)}`,\n toolRow('search', ['harvest_paa', 'search_serp', 'maps_search', 'maps_place_intel'], color),\n toolRow('extract', ['extract_url', 'map_site_urls', 'extract_site', 'audit_site', 'directory_workflow'], color),\n toolRow('build', ['create_editorial_reading_room', 'rank_tracker_workflow', 'portable HTML'], color),\n toolRow('media', ['youtube_harvest', 'youtube_transcribe', 'facebook_ad_search', 'facebook_page_intel', 'facebook_ad_transcribe', 'facebook_video_transcribe', 'instagram_profile_content', 'instagram_media_download', 'reddit_thread'], color),\n toolRow('browser', ['serp_identity_create', 'serp_identity_list', 'browser_open', 'browser_profile_connect', 'browser_profile_list', 'browser_close', 'browser_screenshot', 'browser_read', 'browser_locate', 'browser_replay_mark', 'browser_replay_annotate'], color),\n toolRow('connect', ['list_service_connections', 'describe_service_connection_tool', 'import_service_connection_to_memory', 'export_connected_service_data', 'renew_connected_data_download', 'read_service_connection', 'call_service_connection_action'], color),\n toolRow('commons', ['commons_search_entities', 'commons_get_entity_linkset', 'commons_prepare_entity', 'commons_submit_entity', 'commons_prepare_publication', 'commons_claim_publication', 'commons_publish_editorial', 'commons_get_publication'], color),\n toolRow('account', ['credits_info', 'reports', 'MCP resources'], color),\n toolRow('memory', ['memory-put', 'memory-get', 'memory-search', 'list-vaults', 'record-fact', 'list-scheduled-actions'], color),\n `${colorize('Workflows', 'cyan', color)} ${colorize('(MCP + CLI + API)', 'muted', color)}`,\n toolRow('route', ['workflow_list', 'workflow_suggest', 'workflow_run', 'workflow_step', 'workflow_status', 'workflow_artifact_read'], color),\n toolRow('seo', ['directory', 'agent-packet', 'competitive audit', 'map/serp comparison', 'PAA/AIO briefs', 'scheduled runs'], color),\n '',\n colorize('Usage tips:', 'amber', color),\n 'Run mcp-scraper-install for this visible card. Run mcp-scraper-cli for setup utilities and subcommands.',\n 'Run mcp-scraper in a human terminal to print this card; MCP clients get the same command as a silent stdio server.',\n 'Explicit card command: npx -y -p mcp-scraper@latest mcp-scraper-install',\n 'Hosted browser sessions use direct/no-proxy egress by default.',\n 'Customer auth setup: run browser_profile_connect, send the watch_url, let the user sign in, then call browser_profile_list until AUTHENTICATED.',\n 'Connected account ranges: call export_connected_service_data once. It handles Gmail, Calendar, Zoom, and Resend pagination; do not loop read_service_connection over individual records.',\n 'Connected account RAG: call import_service_connection_to_memory for one bounded approved read. It writes a redacted, untrusted snapshot to a stable Memory path and embeds it for search.',\n 'Stack logins / reconnect: run browser_profile_connect again with the same profile name and another domain to add accounts or refresh a login.',\n 'Start with workflow_suggest for broad jobs like market analysis, ICP research, CRO audits, brand briefs, content gaps, and AI visibility.',\n 'For MCP clients, use mcp-scraper so one install can mix SERP, Maps, browser, reports, and saved MCP resources.',\n 'If you hit the concurrency limit, add 2 browsers for $5/month with mcp-scraper-cli billing concurrency checkout.',\n '',\n `${colorize('Ready.', 'lime', color)} Install the combined MCP server with one command:`,\n '',\n colorize('Setup doctor', 'amber', color),\n 'npx -y -p mcp-scraper@latest mcp-scraper-cli doctor',\n '',\n colorize('Hosted profile setup', 'amber', color),\n 'In your MCP client, call browser_profile_connect with email=\"seo@example.com\" and domain=\"chatgpt.com\".',\n 'Give the returned watch_url to the user. After they sign in, call browser_profile_list, then browser_open with the returned profile. Add more logins by calling browser_profile_connect again with the same profile and a new domain.',\n '',\n colorize('Claude Code one-command setup', 'amber', color),\n claudeCommand,\n 'Then fully exit Claude Code and open a new Claude terminal. Check with: claude mcp list',\n '',\n colorize('Codex config', 'amber', color),\n codexConfig,\n '',\n colorize('Claude Desktop Extension', 'amber', color),\n 'Download: https://mcpscraper.dev/downloads/mcp-scraper.mcpb',\n '',\n colorize('Safety note:', 'muted', color),\n 'mcp-scraper prints this card only when stdin/stdout are an interactive TTY. In MCP clients it writes only JSON-RPC to stdout.',\n 'Use --stdio or MCP_SCRAPER_FORCE_STDIO=1 to force server mode from a terminal.',\n '',\n ].join('\\n')\n}\n"],"mappings":";AAQA,IAAM,QAAwC;AAAA,EAC5C,OAAO;AAAA,EACP,MAAM;AAAA,EACN,MAAM;AAAA,EACN,OAAO;AAAA,EACP,KAAK;AAAA,EACL,OAAO;AAAA,EACP,MAAM;AACR;AAEA,SAAS,SAAS,OAAe,MAAY,SAA0B;AACrE,MAAI,CAAC,QAAS,QAAO;AACrB,SAAO,GAAG,MAAM,IAAI,CAAC,GAAG,KAAK,GAAG,MAAM,KAAK;AAC7C;AAEA,SAAS,QAAQ,OAAe,OAAiB,SAA0B;AACzE,QAAM,SAAS,MAAM,OAAO,GAAG,GAAG;AAClC,SAAO,KAAK,SAAS,QAAQ,SAAS,OAAO,CAAC,IAAI,MAAM,KAAK,SAAS,SAAS,SAAS,OAAO,CAAC,CAAC;AACnG;AAEO,SAAS,sBAAsB,SAAyC;AAC7E,QAAM,QAAQ,QAAQ,SAAS;AAC/B,QAAM,cAAc,QAAQ,mBAAmB,yBAAyB;AACxE,QAAM,QAAQ,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAcrB,QAAM,gBAAgB;AAAA,IACpB,uBAAuB,WAAW;AAAA,IAClC;AAAA,EACF,EAAE,KAAK,IAAI;AAEX,QAAM,cAAc;AAAA,IAClB;AAAA,IACA;AAAA,IACA;AAAA,IACA,kCAAkC,WAAW;AAAA,EAC/C,EAAE,KAAK,IAAI;AAEX,SAAO;AAAA,IACL,SAAS,gBAAgB,QAAQ,OAAO,IAAI,QAAQ,KAAK;AAAA,IACzD,SAAS,yBAAyB,SAAS,KAAK;AAAA,IAChD,SAAS,OAAO,SAAS,KAAK;AAAA,IAC9B,GAAG,SAAS,qBAAqB,QAAQ,KAAK,CAAC,SAAS,QAAQ,OAAO;AAAA,IACvE;AAAA,IACA,SAAS,8KAA8K,QAAQ,KAAK;AAAA,IACpM;AAAA,IACA,GAAG,SAAS,SAAS,QAAQ,KAAK,CAAC,KAAK,SAAS,mBAAmB,SAAS,KAAK,CAAC;AAAA,IACnF,QAAQ,UAAU,CAAC,eAAe,eAAe,eAAe,kBAAkB,GAAG,KAAK;AAAA,IAC1F,QAAQ,WAAW,CAAC,eAAe,iBAAiB,gBAAgB,cAAc,oBAAoB,GAAG,KAAK;AAAA,IAC9G,QAAQ,SAAS,CAAC,iCAAiC,yBAAyB,eAAe,GAAG,KAAK;AAAA,IACnG,QAAQ,SAAS,CAAC,mBAAmB,sBAAsB,sBAAsB,uBAAuB,0BAA0B,6BAA6B,6BAA6B,4BAA4B,eAAe,GAAG,KAAK;AAAA,IAC/O,QAAQ,WAAW,CAAC,wBAAwB,sBAAsB,gBAAgB,2BAA2B,wBAAwB,iBAAiB,sBAAsB,gBAAgB,kBAAkB,uBAAuB,yBAAyB,GAAG,KAAK;AAAA,IACtQ,QAAQ,WAAW,CAAC,4BAA4B,oCAAoC,uCAAuC,iCAAiC,iCAAiC,2BAA2B,gCAAgC,GAAG,KAAK;AAAA,IAChQ,QAAQ,WAAW,CAAC,2BAA2B,8BAA8B,0BAA0B,yBAAyB,+BAA+B,6BAA6B,6BAA6B,yBAAyB,GAAG,KAAK;AAAA,IAC1P,QAAQ,WAAW,CAAC,gBAAgB,WAAW,eAAe,GAAG,KAAK;AAAA,IACtE,QAAQ,UAAU,CAAC,cAAc,cAAc,iBAAiB,eAAe,eAAe,wBAAwB,GAAG,KAAK;AAAA,IAC9H,GAAG,SAAS,aAAa,QAAQ,KAAK,CAAC,KAAK,SAAS,qBAAqB,SAAS,KAAK,CAAC;AAAA,IACzF,QAAQ,SAAS,CAAC,iBAAiB,oBAAoB,gBAAgB,iBAAiB,mBAAmB,wBAAwB,GAAG,KAAK;AAAA,IAC3I,QAAQ,OAAO,CAAC,aAAa,gBAAgB,qBAAqB,uBAAuB,kBAAkB,gBAAgB,GAAG,KAAK;AAAA,IACnI;AAAA,IACA,SAAS,eAAe,SAAS,KAAK;AAAA,IACtC;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,GAAG,SAAS,UAAU,QAAQ,KAAK,CAAC;AAAA,IACpC;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA,SAAS,wBAAwB,SAAS,KAAK;AAAA,IAC/C;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,iCAAiC,SAAS,KAAK;AAAA,IACxD;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA,SAAS,4BAA4B,SAAS,KAAK;AAAA,IACnD;AAAA,IACA;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI;AACb;","names":[]}
1
+ {"version":3,"sources":["../src/install-terminal.ts"],"sourcesContent":["export interface InstallTerminalOptions {\n color?: boolean\n version: string\n apiKeyConfigured?: boolean\n}\n\ntype Tone = 'cyan' | 'lime' | 'amber' | 'red' | 'muted' | 'bold'\n\nconst CODES: Record<Tone | 'reset', string> = {\n reset: '\\x1b[0m',\n cyan: '\\x1b[36m',\n lime: '\\x1b[32m',\n amber: '\\x1b[33m',\n red: '\\x1b[31m',\n muted: '\\x1b[90m',\n bold: '\\x1b[1m',\n}\n\nfunction colorize(value: string, tone: Tone, enabled: boolean): string {\n if (!enabled) return value\n return `${CODES[tone]}${value}${CODES.reset}`\n}\n\nfunction toolRow(label: string, tools: string[], enabled: boolean): string {\n const padded = label.padEnd(9, ' ')\n return ` ${colorize(padded, 'muted', enabled)} ${tools.join(colorize(' . ', 'muted', enabled))}`\n}\n\nexport function renderInstallTerminal(options: InstallTerminalOptions): string {\n const color = options.color ?? true\n const apiKeyValue = options.apiKeyConfigured ? '$MCP_SCRAPER_API_KEY' : 'sk_live_your_key'\n const ascii = String.raw`\n __ __ ____ ____ \n| \\/ |/ ___| _ \\\n| |\\/| | | | |_) |\n| | | | |___| __/\n|_| |_|\\____|_|\n\n ____ ____ ____ _ ____ _____ ____\n/ ___| / ___| _ \\ / \\ | _ \\| ____| _ \\\n\\___ \\| | | |_) | / _ \\ | |_) | _| | |_) |\n ___) | |___| _ < / ___ \\| __/| |___| _ <\n|____/ \\____|_| \\_\\/_/ \\_\\_| |_____|_| \\_\\\n`\n\n const claudeCommand = [\n `MCP_SCRAPER_API_KEY=${apiKeyValue} npx -y -p mcp-scraper@latest \\\\`,\n ' mcp-scraper-cli agent install claude --apply',\n ].join('\\n')\n\n const codexConfig = [\n '[mcp_servers.mcp-scraper]',\n 'command = \"npx\"',\n 'args = [\"-y\", \"-p\", \"mcp-scraper@latest\", \"mcp-scraper\"]',\n `env = { MCP_SCRAPER_API_KEY = \"${apiKeyValue}\" }`,\n ].join('\\n')\n\n return [\n colorize(`mcp-scraper v${options.version}`, 'bold', color),\n colorize('> mcp-scraper-install', 'muted', color),\n colorize(ascii, 'amber', color),\n `${colorize('MCP Scraper Agent', 'cyan', color)} . v${options.version} . mcpscraper.dev`,\n '1/1 install surfaces ready',\n colorize('Newest: any approved connection read can become an indexed Memory snapshot in one call. OAuth stays tenant-isolated and provider content is redacted and marked untrusted.', 'lime', color),\n '',\n `${colorize('Tools', 'cyan', color)} ${colorize('(259 MCP tools)', 'muted', color)}`,\n toolRow('search', ['harvest_paa', 'search_serp', 'maps_search', 'maps_place_intel'], color),\n toolRow('extract', ['extract_url', 'map_site_urls', 'extract_site', 'audit_site', 'directory_workflow'], color),\n toolRow('build', ['create_editorial_reading_room', 'rank_tracker_workflow', 'portable HTML'], color),\n toolRow('media', ['youtube_harvest', 'youtube_transcribe', 'facebook_ad_search', 'facebook_page_intel', 'facebook_ad_transcribe', 'facebook_video_transcribe', 'instagram_profile_content', 'instagram_media_download', 'reddit_thread'], color),\n toolRow('browser', ['serp_identity_create', 'serp_identity_list', 'browser_open', 'browser_profile_connect', 'browser_profile_list', 'browser_close', 'browser_screenshot', 'browser_read', 'browser_locate', 'browser_replay_mark', 'browser_replay_annotate'], color),\n toolRow('connect', ['list_service_connections', 'describe_service_connection_tool', 'import_service_connection_to_memory', 'export_connected_service_data', 'renew_connected_data_download', 'read_service_connection', 'call_service_connection_action'], color),\n toolRow('commons', ['commons_search_entities', 'commons_get_entity_linkset', 'commons_prepare_entity', 'commons_submit_entity', 'commons_prepare_publication', 'commons_claim_publication', 'commons_publish_editorial', 'commons_get_publication'], color),\n toolRow('account', ['credits_info', 'reports', 'MCP resources'], color),\n toolRow('memory', ['memory-put', 'memory-get', 'memory-search', 'list-vaults', 'record-fact', 'list-scheduled-actions'], color),\n `${colorize('Workflows', 'cyan', color)} ${colorize('(MCP + CLI + API)', 'muted', color)}`,\n toolRow('route', ['workflow_list', 'workflow_suggest', 'workflow_run', 'workflow_step', 'workflow_status', 'workflow_artifact_read'], color),\n toolRow('seo', ['directory', 'agent-packet', 'competitive audit', 'map/serp comparison', 'PAA/AIO briefs', 'scheduled runs'], color),\n '',\n colorize('Usage tips:', 'amber', color),\n 'Run mcp-scraper-install for this visible card. Run mcp-scraper-cli for setup utilities and subcommands.',\n 'Run mcp-scraper in a human terminal to print this card; MCP clients get the same command as a silent stdio server.',\n 'Explicit card command: npx -y -p mcp-scraper@latest mcp-scraper-install',\n 'Hosted browser sessions use direct/no-proxy egress by default.',\n 'Customer auth setup: run browser_profile_connect, send the watch_url, let the user sign in, then call browser_profile_list until AUTHENTICATED.',\n 'Connected account ranges: call export_connected_service_data once. It handles Gmail, Calendar, Zoom, and Resend pagination; do not loop read_service_connection over individual records.',\n 'Connected account RAG: call import_service_connection_to_memory for one bounded approved read. It writes a redacted, untrusted snapshot to a stable Memory path and embeds it for search.',\n 'Stack logins / reconnect: run browser_profile_connect again with the same profile name and another domain to add accounts or refresh a login.',\n 'Start with workflow_suggest for broad jobs like market analysis, ICP research, CRO audits, brand briefs, content gaps, and AI visibility.',\n 'For MCP clients, use mcp-scraper so one install can mix SERP, Maps, browser, reports, and saved MCP resources.',\n 'If you hit the concurrency limit, add 2 browsers for $5/month with mcp-scraper-cli billing concurrency checkout.',\n '',\n `${colorize('Ready.', 'lime', color)} Install the combined MCP server with one command:`,\n '',\n colorize('Setup doctor', 'amber', color),\n 'npx -y -p mcp-scraper@latest mcp-scraper-cli doctor',\n '',\n colorize('Hosted profile setup', 'amber', color),\n 'In your MCP client, call browser_profile_connect with email=\"seo@example.com\" and domain=\"chatgpt.com\".',\n 'Give the returned watch_url to the user. After they sign in, call browser_profile_list, then browser_open with the returned profile. Add more logins by calling browser_profile_connect again with the same profile and a new domain.',\n '',\n colorize('Claude Code one-command setup', 'amber', color),\n claudeCommand,\n 'Then fully exit Claude Code and open a new Claude terminal. Check with: claude mcp list',\n '',\n colorize('Codex config', 'amber', color),\n codexConfig,\n '',\n colorize('Claude Desktop Extension', 'amber', color),\n 'Download: https://mcpscraper.dev/downloads/mcp-scraper.mcpb',\n '',\n colorize('Safety note:', 'muted', color),\n 'mcp-scraper prints this card only when stdin/stdout are an interactive TTY. In MCP clients it writes only JSON-RPC to stdout.',\n 'Use --stdio or MCP_SCRAPER_FORCE_STDIO=1 to force server mode from a terminal.',\n '',\n ].join('\\n')\n}\n"],"mappings":";AAQA,IAAM,QAAwC;AAAA,EAC5C,OAAO;AAAA,EACP,MAAM;AAAA,EACN,MAAM;AAAA,EACN,OAAO;AAAA,EACP,KAAK;AAAA,EACL,OAAO;AAAA,EACP,MAAM;AACR;AAEA,SAAS,SAAS,OAAe,MAAY,SAA0B;AACrE,MAAI,CAAC,QAAS,QAAO;AACrB,SAAO,GAAG,MAAM,IAAI,CAAC,GAAG,KAAK,GAAG,MAAM,KAAK;AAC7C;AAEA,SAAS,QAAQ,OAAe,OAAiB,SAA0B;AACzE,QAAM,SAAS,MAAM,OAAO,GAAG,GAAG;AAClC,SAAO,KAAK,SAAS,QAAQ,SAAS,OAAO,CAAC,IAAI,MAAM,KAAK,SAAS,SAAS,SAAS,OAAO,CAAC,CAAC;AACnG;AAEO,SAAS,sBAAsB,SAAyC;AAC7E,QAAM,QAAQ,QAAQ,SAAS;AAC/B,QAAM,cAAc,QAAQ,mBAAmB,yBAAyB;AACxE,QAAM,QAAQ,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAcrB,QAAM,gBAAgB;AAAA,IACpB,uBAAuB,WAAW;AAAA,IAClC;AAAA,EACF,EAAE,KAAK,IAAI;AAEX,QAAM,cAAc;AAAA,IAClB;AAAA,IACA;AAAA,IACA;AAAA,IACA,kCAAkC,WAAW;AAAA,EAC/C,EAAE,KAAK,IAAI;AAEX,SAAO;AAAA,IACL,SAAS,gBAAgB,QAAQ,OAAO,IAAI,QAAQ,KAAK;AAAA,IACzD,SAAS,yBAAyB,SAAS,KAAK;AAAA,IAChD,SAAS,OAAO,SAAS,KAAK;AAAA,IAC9B,GAAG,SAAS,qBAAqB,QAAQ,KAAK,CAAC,SAAS,QAAQ,OAAO;AAAA,IACvE;AAAA,IACA,SAAS,8KAA8K,QAAQ,KAAK;AAAA,IACpM;AAAA,IACA,GAAG,SAAS,SAAS,QAAQ,KAAK,CAAC,KAAK,SAAS,mBAAmB,SAAS,KAAK,CAAC;AAAA,IACnF,QAAQ,UAAU,CAAC,eAAe,eAAe,eAAe,kBAAkB,GAAG,KAAK;AAAA,IAC1F,QAAQ,WAAW,CAAC,eAAe,iBAAiB,gBAAgB,cAAc,oBAAoB,GAAG,KAAK;AAAA,IAC9G,QAAQ,SAAS,CAAC,iCAAiC,yBAAyB,eAAe,GAAG,KAAK;AAAA,IACnG,QAAQ,SAAS,CAAC,mBAAmB,sBAAsB,sBAAsB,uBAAuB,0BAA0B,6BAA6B,6BAA6B,4BAA4B,eAAe,GAAG,KAAK;AAAA,IAC/O,QAAQ,WAAW,CAAC,wBAAwB,sBAAsB,gBAAgB,2BAA2B,wBAAwB,iBAAiB,sBAAsB,gBAAgB,kBAAkB,uBAAuB,yBAAyB,GAAG,KAAK;AAAA,IACtQ,QAAQ,WAAW,CAAC,4BAA4B,oCAAoC,uCAAuC,iCAAiC,iCAAiC,2BAA2B,gCAAgC,GAAG,KAAK;AAAA,IAChQ,QAAQ,WAAW,CAAC,2BAA2B,8BAA8B,0BAA0B,yBAAyB,+BAA+B,6BAA6B,6BAA6B,yBAAyB,GAAG,KAAK;AAAA,IAC1P,QAAQ,WAAW,CAAC,gBAAgB,WAAW,eAAe,GAAG,KAAK;AAAA,IACtE,QAAQ,UAAU,CAAC,cAAc,cAAc,iBAAiB,eAAe,eAAe,wBAAwB,GAAG,KAAK;AAAA,IAC9H,GAAG,SAAS,aAAa,QAAQ,KAAK,CAAC,KAAK,SAAS,qBAAqB,SAAS,KAAK,CAAC;AAAA,IACzF,QAAQ,SAAS,CAAC,iBAAiB,oBAAoB,gBAAgB,iBAAiB,mBAAmB,wBAAwB,GAAG,KAAK;AAAA,IAC3I,QAAQ,OAAO,CAAC,aAAa,gBAAgB,qBAAqB,uBAAuB,kBAAkB,gBAAgB,GAAG,KAAK;AAAA,IACnI;AAAA,IACA,SAAS,eAAe,SAAS,KAAK;AAAA,IACtC;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,GAAG,SAAS,UAAU,QAAQ,KAAK,CAAC;AAAA,IACpC;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA,SAAS,wBAAwB,SAAS,KAAK;AAAA,IAC/C;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,iCAAiC,SAAS,KAAK;AAAA,IACxD;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA,SAAS,4BAA4B,SAAS,KAAK;AAAA,IACnD;AAAA,IACA;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI;AACb;","names":[]}
@@ -17,7 +17,7 @@ import {
17
17
  } from "./chunk-4LINSJNI.js";
18
18
  import {
19
19
  PACKAGE_VERSION
20
- } from "./chunk-DYSXI6QU.js";
20
+ } from "./chunk-5BODYBIP.js";
21
21
  import {
22
22
  createPrivateArtifact,
23
23
  privateArtifactOwnerId,
@@ -30,7 +30,7 @@ import {
30
30
  PAA_QUESTION_CREDITS,
31
31
  PAGE_SCRAPE_CREDITS,
32
32
  SERP_SEARCH_CREDITS
33
- } from "./chunk-TRSZEK2D.js";
33
+ } from "./chunk-2XUXVKT4.js";
34
34
  import {
35
35
  PUBLIC_ERROR_CODES,
36
36
  buildPublicErrorEnvelope,
@@ -342,6 +342,8 @@ seam is noted so you can chain them.
342
342
  - Whole site -> **extract_site** (takes a url). It durably retains complete per-page JSON, acquired HTML,
343
343
  and Markdown. Poll **check_site_export**, then call **site_export_read** for the manifest or a page view;
344
344
  call **site_export_image** for downloaded image IDs.
345
+ - JavaScript-rendered page overlap/cannibalization -> **analyze_site_similarity**. It returns dedicated
346
+ raw-cosine page pairs, corpus percentiles, threshold clusters, and rendered-page artifacts.
345
347
  - Wayback replay URLs work with the same tools: \`extract_url\` removes playback chrome and can return
346
348
  a featured image; \`extract_site\` batches nearby archived HTML captures for the replayed site.
347
349
  - For multiple archive months, pass \`extract_site.wayback\` with explicit \`months\` or a \`from\`/\`to\`
@@ -2589,7 +2591,7 @@ async function formatExtractSite(raw, input, ctx) {
2589
2591
  const parsed = parseData(raw);
2590
2592
  if ("error" in parsed) return formattedErrorResult(parsed.error);
2591
2593
  const started = formatBackgroundJobStarted(
2592
- "Multi-Page Site Content Crawl",
2594
+ input.toolLabel ?? "Multi-Page Site Content Crawl",
2593
2595
  parsed.data,
2594
2596
  {
2595
2597
  requested: input.delivery ?? "auto",
@@ -4140,7 +4142,8 @@ function formatLeadListImport(raw) {
4140
4142
  const map = Object.keys(detailedMap).length > 0 ? detailedMap : Object.fromEntries(Object.entries(simpleMap).map(([field, header]) => [field, {
4141
4143
  header: header == null ? null : String(header),
4142
4144
  confidence: header == null ? 0 : 1,
4143
- reason: "server_suggestion"
4145
+ reason: "server_suggestion",
4146
+ accepted: header != null
4144
4147
  }]));
4145
4148
  const mappingLines = Object.entries(map).slice(0, 12).map(([field, suggestion]) => {
4146
4149
  const record = structuredRecord(suggestion);
@@ -4995,6 +4998,7 @@ ${rows}` : ""
4995
4998
  var ESSENTIAL_OUTPUT_SCHEMA_TOOLS = /* @__PURE__ */ new Set([
4996
4999
  "extract_url",
4997
5000
  "extract_site",
5001
+ "analyze_site_similarity",
4998
5002
  "audit_site",
4999
5003
  "check_site_export",
5000
5004
  "site_export_read",
@@ -6311,6 +6315,13 @@ var ExtractSiteInputSchema = {
6311
6315
  preserveMedia: z5.boolean().default(false).describe("Include supported images in the export bundle. This is the preferred replacement for downloadImages."),
6312
6316
  downloadImages: z5.boolean().optional().describe("Deprecated alias for preserveMedia. Omit when using preserveMedia; when omitted, image preservation defaults to false.")
6313
6317
  };
6318
+ var AnalyzeSiteSimilarityInputSchema = {
6319
+ url: WebsiteUrlOrDomainSchema.describe("Public live site to render and compare. Bare domains default to https://."),
6320
+ maxPages: z5.number().int().min(2).max(500).default(100).describe("Maximum rendered pages to compare; default 100, maximum 500."),
6321
+ similarityThreshold: z5.number().min(0).max(1).default(0.9).describe("Minimum raw cosine score retained in the pair table; default 0.90."),
6322
+ similarityMaxPairs: z5.number().int().min(1).max(5e4).default(1e4).describe("Maximum scored pairs retained, highest first; default 10,000."),
6323
+ idempotencyKey: z5.string().trim().min(8).max(200).describe("Required unique opaque ID for this intended analysis. Reuse only when retrying the same call; use a new value for an intentional rerun.")
6324
+ };
6314
6325
  var AuditSiteInputSchema = {
6315
6326
  url: WebsiteUrlOrDomainSchema.describe("Public website URL or domain for a full technical SEO audit (issues, link graph, indexability, headings, images). Bare domains default to https://. For plain content use extract_site instead."),
6316
6327
  maxPages: z5.number().int().min(1).max(1e4).optional().describe("Maximum pages to crawl and audit. MCP audits always run as durable background exports and return a jobId; poll check_site_export for the hosted audit ZIP."),
@@ -6323,17 +6334,17 @@ var AuditSiteInputSchema = {
6323
6334
  downloadImages: z5.boolean().optional().describe("Deprecated alias for preserveMedia. Omit when using preserveMedia; when omitted, image preservation defaults to false.")
6324
6335
  };
6325
6336
  var CheckSiteExportInputSchema = {
6326
- jobId: z5.string().min(1).describe("The jobId returned by extract_site or audit_site. Poll until status is complete, partial, or failed; partial jobs still return a downloadable bundle with successful pages and failure details.")
6337
+ jobId: z5.string().min(1).describe("The jobId returned by extract_site, analyze_site_similarity, or audit_site. Poll until status is complete, partial, or failed; partial jobs still return successful content and failure details.")
6327
6338
  };
6328
6339
  var SiteExportReadInputSchema = {
6329
- jobId: z5.string().min(1).describe("Site export job ID returned by extract_site or audit_site."),
6340
+ jobId: z5.string().min(1).describe("Site export job ID returned by extract_site, analyze_site_similarity, or audit_site."),
6330
6341
  pageId: z5.string().regex(/^[a-f0-9]{64}$/).optional().describe("Page ID returned by a manifest read. Omit to list the export manifest."),
6331
6342
  format: z5.enum(["manifest", "json", "html", "markdown"]).default("manifest").describe("manifest lists page/image IDs; JSON, HTML, and Markdown read one page representation."),
6332
6343
  offset: z5.number().int().min(0).default(0).describe("UTF-8 byte offset. Continue from nextOffset until it is null."),
6333
6344
  maxBytes: z5.number().int().min(1).max(1e6).default(64e3).describe("Maximum UTF-8 bytes returned in this window.")
6334
6345
  };
6335
6346
  var SiteExportImageInputSchema = {
6336
- jobId: z5.string().min(1).describe("Site export job ID returned by extract_site or audit_site."),
6347
+ jobId: z5.string().min(1).describe("Site export job ID returned by extract_site, analyze_site_similarity, or audit_site."),
6337
6348
  imageId: z5.string().regex(/^[a-f0-9]{64}$/).describe("Downloaded image ID returned by a site_export_read manifest.")
6338
6349
  };
6339
6350
  var ArchiveReadInputSchema = {
@@ -7161,7 +7172,8 @@ var DirectoryWorkflowOutputSchema = {
7161
7172
  var LeadSuggestedColumnOutputSchema = z5.object({
7162
7173
  header: NullableString,
7163
7174
  confidence: z5.number().min(0).max(1),
7164
- reason: z5.string()
7175
+ reason: z5.string(),
7176
+ accepted: z5.boolean()
7165
7177
  }).strict();
7166
7178
  var LeadColumnMapSuggestionOutputSchema = z5.object({
7167
7179
  name: LeadSuggestedColumnOutputSchema,
@@ -10207,6 +10219,21 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
10207
10219
  await formatExtractSite(await executor.extractSite(input), input, ctx),
10208
10220
  requestContext
10209
10221
  ));
10222
+ server.registerTool("analyze_site_similarity", {
10223
+ title: "Rendered Site Content Similarity",
10224
+ description: "Find overlapping or competing live pages. Forces JavaScript rendering, retains bounded inert DOM plus clean Markdown, and returns raw-cosine pairs, corpus percentiles, clusters, and table-ready files in a private ZIP. Poll check_site_export; the scores measure content overlap, not intent or proof of cannibalization.",
10225
+ inputSchema: AnalyzeSiteSimilarityInputSchema,
10226
+ outputSchema: recordOutputSchema("analyze_site_similarity", ExtractSiteOutputSchema),
10227
+ annotations: { ...liveWebToolAnnotations("Rendered Site Content Similarity"), readOnlyHint: false }
10228
+ }, async (input, requestContext) => tasks.taskify(
10229
+ "site_export",
10230
+ await formatExtractSite(
10231
+ await executor.analyzeSiteSimilarity(input),
10232
+ { ...input, toolLabel: "Rendered Site Content Similarity" },
10233
+ ctx
10234
+ ),
10235
+ requestContext
10236
+ ));
10210
10237
  server.registerTool("audit_site", {
10211
10238
  title: "Technical SEO Audit",
10212
10239
  description: `Run a full technical SEO audit (Screaming-Frog-style) on a public website: on-page issues, internal link graph, indexability, heading/image analysis. Pass a new idempotencyKey for each intended audit and reuse it only when retrying that call. Every MCP audit starts a durable export; poll check_site_export for discovered, attempted, successful, failed, and remaining counts plus ${fileBehavior("the saved ZIP.", "the owner-scoped downloadable ZIP.")} Use extract_site instead for plain page content.`,
@@ -10220,7 +10247,7 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
10220
10247
  ));
10221
10248
  server.registerTool("check_site_export", {
10222
10249
  title: "Check Site Export",
10223
- description: "Poll a background extract_site or audit_site job. Reports discovered, attempted, successful, failed, and remaining pages plus a stable public error envelope when terminal. Complete and partial jobs expose direct AI readback through site_export_read/site_export_image and also return a downloadable ZIP.",
10250
+ description: "Poll a background extract_site, analyze_site_similarity, or audit_site job. Reports page counters and a stable public error envelope when terminal. Complete and partial jobs expose direct AI readback and a downloadable ZIP.",
10224
10251
  inputSchema: CheckSiteExportInputSchema,
10225
10252
  outputSchema: recordOutputSchema("check_site_export", CheckSiteExportOutputSchema),
10226
10253
  annotations: { ...liveWebToolAnnotations("Check Site Export"), readOnlyHint: false }
@@ -11219,6 +11246,19 @@ var HttpMcpToolExecutor = class {
11219
11246
  const { idempotencyKey, ...body } = input;
11220
11247
  return this.callSiteExtractStart("extract_site", { ...body, background: true }, idempotencyKey);
11221
11248
  }
11249
+ analyzeSiteSimilarity(input) {
11250
+ const { idempotencyKey, similarityThreshold, similarityMaxPairs, ...body } = input;
11251
+ return this.callSiteExtractStart("analyze_site_similarity", {
11252
+ ...body,
11253
+ background: true,
11254
+ formats: ["markdown", "links", "json"],
11255
+ renderJavaScript: true,
11256
+ captureRenderedDom: true,
11257
+ semanticSimilarity: true,
11258
+ similarityThreshold,
11259
+ similarityMaxPairs
11260
+ }, idempotencyKey);
11261
+ }
11222
11262
  auditSite(input) {
11223
11263
  const { idempotencyKey, ...body } = input;
11224
11264
  const requestBody = {
@@ -16670,4 +16710,4 @@ export {
16670
16710
  ScheduledResultsMcpExecutor,
16671
16711
  registerScheduledResultsMcpTools
16672
16712
  };
16673
- //# sourceMappingURL=chunk-EEW3DMI5.js.map
16713
+ //# sourceMappingURL=chunk-PSKRQDGN.js.map