mcp-scraper 0.64.0 → 0.65.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/README.md +2 -2
- package/dist/bin/api-server.cjs +1028 -497
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +2 -2
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +2 -2
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +2 -2
- package/dist/bin/mcp-stdio-server.cjs +49 -9
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +4 -4
- package/dist/{chunk-TRSZEK2D.js → chunk-2XUXVKT4.js} +3 -3
- package/dist/{chunk-TRSZEK2D.js.map → chunk-2XUXVKT4.js.map} +1 -1
- package/dist/chunk-5BODYBIP.js +7 -0
- package/dist/chunk-5BODYBIP.js.map +1 -0
- package/dist/{chunk-BRKCPONM.js → chunk-5PIO7QBG.js} +2 -2
- package/dist/{chunk-5QYSY5KI.js → chunk-AMLFKPLL.js} +139 -3
- package/dist/chunk-AMLFKPLL.js.map +1 -0
- package/dist/{chunk-3NXLGTC6.js → chunk-DCWXVAQT.js} +2 -2
- package/dist/{chunk-3NXLGTC6.js.map → chunk-DCWXVAQT.js.map} +1 -1
- package/dist/{chunk-EEW3DMI5.js → chunk-PSKRQDGN.js} +50 -10
- package/dist/chunk-PSKRQDGN.js.map +1 -0
- package/dist/{extract-bundle-FW23CEMG.js → extract-bundle-UWKJT4MU.js} +371 -14
- package/dist/extract-bundle-UWKJT4MU.js.map +1 -0
- package/dist/{server-TMAWQYZE.js → server-AFEA235I.js} +170 -163
- package/dist/server-AFEA235I.js.map +1 -0
- package/dist/{site-extract-repository-UXSFD2QM.js → site-extract-repository-J3RH3MOS.js} +3 -3
- package/dist/{worker-DD243CON.js → worker-YKRIK5LS.js} +2 -2
- package/package.json +3 -1
- package/dist/chunk-5QYSY5KI.js.map +0 -1
- package/dist/chunk-DYSXI6QU.js +0 -7
- package/dist/chunk-DYSXI6QU.js.map +0 -1
- package/dist/chunk-EEW3DMI5.js.map +0 -1
- package/dist/extract-bundle-FW23CEMG.js.map +0 -1
- package/dist/server-TMAWQYZE.js.map +0 -1
- /package/dist/{chunk-BRKCPONM.js.map → chunk-5PIO7QBG.js.map} +0 -0
- /package/dist/{site-extract-repository-UXSFD2QM.js.map → site-extract-repository-J3RH3MOS.js.map} +0 -0
- /package/dist/{worker-DD243CON.js.map → worker-YKRIK5LS.js.map} +0 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/lib/media-extractor.ts","../src/api/extraction-problems.ts","../src/api/commons-embeddings.ts"],"sourcesContent":["import { createWriteStream, mkdirSync, rmSync } from 'node:fs'\nimport { homedir } from 'node:os'\nimport { join, extname, basename } from 'node:path'\nimport { pipeline } from 'node:stream/promises'\nimport { Readable, Transform } from 'node:stream'\nimport { validatePublicHttpUrl } from '../api/url-utils.js'\nimport type { RenderedMediaDiscovery } from './rendered-media.js'\nimport { loadHtmlDocument, type HtmlDocument } from './html-document.js'\n\nconst AD_PATTERNS: string[] = [\n 'doubleclick.net', 'googlesyndication.com', 'googletagmanager.com',\n 'google-analytics.com', 'googletagservices.com', 'adservice.google',\n 'googletag.', 'pagead2.googlesyndication',\n 'facebook.net/tr', 'connect.facebook.net', 'fbcdn.net/rsrc',\n 'analytics.twitter.com', 'static.ads-twitter.com', 'ads.twitter.com',\n 't.co/i/adsct', 'pixel.advertising.com',\n 'hotjar.com', 'clarity.ms', 'quantserve.com', 'scorecardresearch.com',\n 'newrelic.com', 'nr-data.net', 'segment.io', 'segment.com',\n 'amplitude.com', 'mixpanel.com', 'heap.io', 'fullstory.com',\n 'moatads.com', 'criteo.com', 'adsrvr.org', 'rubiconproject.com',\n 'pubmatic.com', 'openx.net', 'appnexus.com', 'amazon-adsystem.com',\n 'media.net', 'yieldmo.com', 'triplelift.com', 'sharethrough.com',\n 'prebid.', 'smaato.net', 'indexworm.com', 'casalemedia.com',\n 'outbrain.com', 'taboola.com', 'revcontent.com', 'mgid.com',\n 'tawk.to', 'intercom.io', 'drift.com', 'hs-scripts.com',\n 'zopim.com', 'livechatinc.com', 'userlike.com',\n 'onetrust.com', 'cookielaw.org', 'cookieinformation.com', 'trustarc.com',\n '/ads/', '/ad/', '/banner/', '/banners/', '/pixel/', '/beacon/',\n '/tracking/', '/tracker/', '/remarketing/', '/conversion/',\n '1x1.gif', 'spacer.gif', 'blank.gif', 'transparent.gif',\n]\n\nconst IMAGE_EXTS = new Set(['.jpg', '.jpeg', '.png', '.webp', '.gif', '.avif', '.svg', '.tiff'])\nconst VIDEO_EXTS = new Set(['.mp4', '.webm', '.mov', '.avi', '.m4v', '.ogv', '.mkv'])\nconst AUDIO_EXTS = new Set(['.mp3', '.wav', '.ogg', '.aac', '.m4a', '.flac', '.opus'])\n\nexport type MediaType = 'image' | 'video' | 'audio'\n\nexport interface MediaAsset {\n url: string\n type: MediaType\n mimeType: string | null\n filename: string\n savedPath: string | null\n sizeBytes: number | null\n discoveryMethods: string[]\n altTexts: string[]\n contexts: string[]\n width: number | null\n height: number | null\n variants: string[]\n finalUrl?: string | null\n duplicateOf?: string | null\n sha256?: string | null\n downloadStatus?: 'downloaded' | 'failed' | 'not_attempted'\n downloadError?: string | null\n inlinePreview?: { data: string; mimeType: string } | null\n}\n\nexport interface MediaManifest {\n pageUrl: string\n outputDir: string | null\n assets: MediaAsset[]\n filteredCount: number\n totalFound: number\n staticFound: number\n renderedFound: number\n completeness: 'complete' | 'partial'\n exhausted: boolean\n stopReason: 'page_exhausted' | 'asset_limit' | 'scroll_round_limit' | 'render_unavailable'\n scrollRounds: number\n warnings: string[]\n artifact: {\n artifactId: string\n filename: string\n contentType: string\n bytes: number\n sha256: string\n expiresAt: string\n downloadUrl: string | null\n downloadUrlExpiresAt: string | null\n localPath: string | null\n } | null\n}\n\nexport interface MediaExtractOptions {\n types?: MediaType[]\n outputDir?: string | null\n rendered?: RenderedMediaDiscovery | null\n maxAssets?: number\n}\n\nexport interface DiscoveredMediaCandidate {\n url: string\n type: MediaType | null\n discoveryMethods: string[]\n altTexts: string[]\n contexts: string[]\n width: number | null\n height: number | null\n}\n\nfunction isAdUrl(url: string): boolean {\n const lower = url.toLowerCase()\n return AD_PATTERNS.some(p => lower.includes(p))\n}\n\nfunction isDataUri(url: string): boolean {\n return url.startsWith('data:')\n}\n\nexport function typeFromUrl(url: string): MediaType | null {\n try {\n const ext = extname(new URL(url).pathname).toLowerCase()\n if (IMAGE_EXTS.has(ext)) return 'image'\n if (VIDEO_EXTS.has(ext)) return 'video'\n if (AUDIO_EXTS.has(ext)) return 'audio'\n } catch { /* non-parseable URL */ }\n return null\n}\n\nexport function typeFromMime(mime: string): MediaType | null {\n const lower = mime.toLowerCase()\n if (lower.startsWith('image/')) return 'image'\n if (lower.startsWith('video/')) return 'video'\n if (lower.startsWith('audio/')) return 'audio'\n return null\n}\n\nfunction resolveUrl(raw: string, base: string): string | null {\n if (!raw || isDataUri(raw)) return null\n try { return new URL(raw, base).href } catch { return null }\n}\n\nfunction safeFilename(url: string, index: number): string {\n try {\n const u = new URL(url)\n const base = basename(u.pathname).replace(/[^a-zA-Z0-9._-]/g, '_').slice(0, 80)\n return base || `asset-${index}`\n } catch {\n return `asset-${index}`\n }\n}\n\nfunction boundedText(value: string | undefined | null, max: number): string | null {\n const normalized = value?.replace(/&(?:amp|#0?38|#x26);/gi, '&').replace(/\\s+/g, ' ').trim()\n return normalized ? normalized.slice(0, max) : null\n}\n\nfunction dimensionsFromUrl(rawUrl: string): { width: number; height: number } | null {\n const match = rawUrl.match(/[-_/](\\d{2,5})x(\\d{2,5})(?=[._/?#-]|$)/i)\n return match ? { width: Number(match[1]), height: Number(match[2]) } : null\n}\n\nexport function mediaFamilyKey(rawUrl: string): string {\n try {\n const url = new URL(rawUrl)\n url.hash = ''\n url.pathname = url.pathname.replace(/-(?:\\d{2,5}x\\d{2,5}|scaled|e\\d{6,})(?=\\.[a-z0-9]{2,6}$)/i, '')\n for (const key of ['w', 'width', 'h', 'height', 'resize']) url.searchParams.delete(key)\n url.searchParams.sort()\n return url.href\n } catch {\n return rawUrl\n }\n}\n\nfunction candidateScore(candidate: DiscoveredMediaCandidate): number {\n const area = (candidate.width ?? 0) * (candidate.height ?? 0)\n const methodBoost = candidate.discoveryMethods.some(method => method.includes('lightbox')) ? 1_000_000_000_000 : 0\n const originalBoost = /-(?:\\d{2,5}x\\d{2,5}|scaled|e\\d{6,})(?=\\.[a-z0-9]{2,6}(?:[?#]|$))/i.test(candidate.url) ? 0 : 100_000_000_000\n return methodBoost + originalBoost + area\n}\n\nfunction mergeCandidate(target: Map<string, DiscoveredMediaCandidate>, incoming: DiscoveredMediaCandidate): void {\n const existing = target.get(incoming.url)\n if (!existing) {\n target.set(incoming.url, incoming)\n return\n }\n existing.discoveryMethods = [...new Set([...existing.discoveryMethods, ...incoming.discoveryMethods])]\n existing.altTexts = [...new Set([...existing.altTexts, ...incoming.altTexts])]\n existing.contexts = [...new Set([...existing.contexts, ...incoming.contexts])]\n if (!existing.type && incoming.type) existing.type = incoming.type\n if ((incoming.width ?? 0) * (incoming.height ?? 0) > (existing.width ?? 0) * (existing.height ?? 0)) {\n existing.width = incoming.width\n existing.height = incoming.height\n }\n}\n\nexport function discoverStaticMedia(html: string | HtmlDocument, baseUrl: string): DiscoveredMediaCandidate[] {\n const found = new Map<string, DiscoveredMediaCandidate>()\n const add = (\n raw: string | undefined,\n method: string,\n explicitType: MediaType | null = null,\n alt?: string | null,\n context?: string | null,\n dimensions?: { width: number; height: number } | null,\n ) => {\n const resolved = raw ? resolveUrl(raw.trim().replace(/&/g, '&'), baseUrl) : null\n if (!resolved || isAdUrl(resolved)) return\n const inferredDimensions = dimensions ?? dimensionsFromUrl(resolved)\n mergeCandidate(found, {\n url: resolved,\n type: explicitType ?? typeFromUrl(resolved),\n discoveryMethods: [method],\n altTexts: alt ? [alt] : [],\n contexts: context ? [context] : [],\n width: inferredDimensions?.width ?? null,\n height: inferredDimensions?.height ?? null,\n })\n }\n const scanSrcset = (value: string | undefined, method: string, type: MediaType, alt?: string | null) => {\n for (const part of (value ?? '').split(',')) {\n const [url, descriptor] = part.trim().split(/\\s+/)\n if (!url) continue\n const width = descriptor?.match(/^(\\d+)w$/)?.[1]\n add(url, method, type, alt, null, width ? { width: Number(width), height: 0 } : null)\n }\n }\n const scan = (document: HtmlDocument, suffix = '') => {\n const $ = document.$\n for (const image of document.images) {\n const attrs = image.attributes\n const alt = boundedText(image.alt, 500)\n const dimensions = image.width && image.height ? { width: image.width, height: image.height } : null\n for (const attr of ['src', 'data-src', 'data-lazy-src', 'data-original', 'data-bg', 'data-background', 'data-bg-url', 'data-lazy-bg', 'data-echo']) {\n const value = attrs[attr]\n add(value, `static-${attr}${suffix}`, 'image', alt, null, dimensions)\n }\n scanSrcset(attrs.srcset, `static-srcset${suffix}`, 'image', alt)\n scanSrcset(attrs['data-srcset'], `static-data-srcset${suffix}`, 'image', alt)\n }\n $('source').each((_index, element) => {\n const node = $(element)\n const declared = node.attr('type') ?? ''\n const type: MediaType = declared.startsWith('video/') ? 'video' : declared.startsWith('audio/') ? 'audio' : 'image'\n add(node.attr('src'), `static-source${suffix}`, type)\n scanSrcset(node.attr('srcset'), `static-source-srcset${suffix}`, type)\n })\n $('video,audio').each((_index, element) => {\n const node = $(element)\n const type = element.tagName.toLowerCase() as 'video' | 'audio'\n add(node.attr('src'), `static-${type}${suffix}`, type)\n if (type === 'video') add(node.attr('poster'), `static-video-poster${suffix}`, 'image')\n })\n $('meta').each((_index, element) => {\n const node = $(element)\n const prop = (node.attr('property') ?? node.attr('name') ?? '').toLowerCase()\n if (prop === 'og:image' || prop === 'og:image:url' || prop === 'twitter:image' || prop === 'twitter:image:src') {\n add(node.attr('content'), `static-${prop}${suffix}`, 'image')\n }\n })\n $('link[href]').each((_index, element) => {\n const node = $(element)\n const rel = (node.attr('rel') ?? '').toLowerCase().split(/\\s+/)\n if (rel.some(value => value === 'icon' || value === 'apple-touch-icon')) {\n add(node.attr('href'), `static-icon${suffix}`, 'image')\n }\n })\n $('svg image').each((_index, element) => {\n const node = $(element)\n add(node.attr('href') ?? node.attr('xlink:href'), `static-svg-image${suffix}`, 'image')\n })\n $('a[href]').has('img').each((_index, element) => {\n const node = $(element)\n const href = node.attr('href')\n if (!href) return\n if (\n /lightbox|gallery|fancybox|glightbox|swipebox|elementor-open-lightbox/i.test(`${node.attr('class') ?? ''} ${node.attr('data-elementor-open-lightbox') ?? ''}`)\n || /\\.(?:avif|gif|jpe?g|png|svg|webp)(?:[?#]|$)/i.test(href)\n ) {\n const alt = boundedText(node.find('img').first().attr('alt'), 500)\n add(href, `static-lightbox-target${suffix}`, 'image', alt)\n }\n })\n const cssSources = $('style').toArray().map(element => $(element).text())\n $('[style]').each((_index, element) => { cssSources.push($(element).attr('style') ?? '') })\n for (const match of cssSources.join('\\n').matchAll(/(?:background(?:-image)?\\s*:\\s*)?url\\(\\s*(?:[\"']|"|�?34;|'|�?39;)?([^\"')&]+)(?:[\"']|"|�?34;|'|�?39;)?\\s*\\)/gi)) {\n add(match[1], `static-css-url${suffix}`, 'image')\n }\n for (const match of document.sourceHtml.matchAll(/[\"'](https?:\\/\\/[^\"'\\s<>]+\\.(?:avif|gif|jpe?g|png|svg|webp|mp4|webm|mp3|ogg)(?:\\?[^\"'\\s<>]*)?)[\"']/gi)) {\n add(match[1], `static-bare-url${suffix}`)\n }\n }\n\n const document = typeof html === 'string' ? loadHtmlDocument(html) : html\n scan(document)\n document.$('script').each((_index, element) => {\n const source = document.$(element).text()\n const unescaped = source\n .replace(/\\\\u003[cC]/g, '<').replace(/\\\\u003[eE]/g, '>').replace(/\\\\u0026/gi, '&')\n .replace(/\\\\[\"']/g, match => match.slice(1)).replace(/\\\\\\//g, '/')\n if (unescaped !== source && /<(?:img|source|video|audio)\\b/i.test(unescaped)) {\n scan(loadHtmlDocument(unescaped), '-json-unescaped')\n }\n })\n return [...found.values()]\n}\n\nexport function mergeMediaDiscovery(\n staticAssets: DiscoveredMediaCandidate[],\n rendered: RenderedMediaDiscovery | null | undefined,\n maxAssets = 100,\n): Array<DiscoveredMediaCandidate & { variants: string[] }> {\n const exact = new Map<string, DiscoveredMediaCandidate>()\n for (const asset of staticAssets) mergeCandidate(exact, asset)\n for (const asset of rendered?.assets ?? []) mergeCandidate(exact, { ...asset, type: asset.type })\n\n const families = new Map<string, DiscoveredMediaCandidate[]>()\n for (const asset of exact.values()) {\n const key = mediaFamilyKey(asset.url)\n const family = families.get(key) ?? []\n family.push(asset)\n families.set(key, family)\n }\n return [...families.values()]\n .map(family => {\n const ordered = [...family].sort((a, b) => candidateScore(b) - candidateScore(a) || a.url.localeCompare(b.url))\n const preferred = { ...ordered[0] }\n preferred.discoveryMethods = [...new Set(family.flatMap(asset => asset.discoveryMethods))]\n preferred.altTexts = [...new Set(family.flatMap(asset => asset.altTexts))]\n preferred.contexts = [...new Set(family.flatMap(asset => asset.contexts))]\n return { ...preferred, variants: ordered.slice(1).map(asset => asset.url) }\n })\n .sort((a, b) => candidateScore(b) - candidateScore(a) || a.url.localeCompare(b.url))\n .slice(0, Math.max(1, maxAssets))\n}\n\nexport function extractMediaUrls(html: string, baseUrl: string): string[] {\n return mergeMediaDiscovery(discoverStaticMedia(html, baseUrl), null, 2_000).map(asset => asset.url)\n}\n\nexport async function downloadAsset(\n url: string,\n destDir: string,\n filename: string,\n options: {\n expectedType?: MediaType\n maxBytes?: number\n consumeBytes?: (bytes: number) => boolean\n } = {},\n): Promise<{ savedPath: string; sizeBytes: number; mimeType: string | null }> {\n const maxBytes = Math.max(1, Math.min(options.maxBytes ?? 50 * 1024 * 1024, 100 * 1024 * 1024))\n let target = url\n let res: Response | null = null\n for (let redirects = 0; redirects <= 5; redirects++) {\n const checked = await validatePublicHttpUrl(target, { field: 'media URL' })\n if (checked.error || !checked.parsed) throw new Error(checked.error ?? 'Media URL was rejected')\n res = await fetch(checked.parsed.href, {\n signal: AbortSignal.timeout(15_000),\n redirect: 'manual',\n })\n if (res.status >= 300 && res.status < 400) {\n const location = res.headers.get('location')\n if (!location) throw new Error(`HTTP ${res.status} redirect did not include Location`)\n target = new URL(location, checked.parsed.href).href\n res = null\n continue\n }\n break\n }\n if (!res) throw new Error('Media download exceeded five redirects')\n if (!res.ok) throw new Error(`HTTP ${res.status}`)\n if (!res.body) throw new Error('Empty response body')\n\n const mimeType = res.headers.get('content-type')?.split(';')[0].trim() ?? null\n if (options.expectedType && (!mimeType || typeFromMime(mimeType) !== options.expectedType)) {\n throw new Error(`Expected ${options.expectedType} content but received ${mimeType ?? 'no content-type'}`)\n }\n const declaredBytes = Number(res.headers.get('content-length') ?? 0)\n if (Number.isFinite(declaredBytes) && declaredBytes > maxBytes) {\n throw new Error(`Media asset exceeds ${maxBytes} byte limit`)\n }\n\n let dest = join(destDir, filename)\n if (mimeType && !extname(filename)) {\n const mimeExt: Record<string, string> = {\n 'image/jpeg': '.jpg', 'image/png': '.png', 'image/webp': '.webp',\n 'image/gif': '.gif', 'image/svg+xml': '.svg', 'image/avif': '.avif',\n 'video/mp4': '.mp4', 'video/webm': '.webm',\n 'audio/mpeg': '.mp3', 'audio/ogg': '.ogg', 'audio/wav': '.wav',\n }\n const ext = mimeExt[mimeType]\n if (ext) dest = dest + ext\n }\n\n const writer = createWriteStream(dest)\n await new Promise<void>((resolve, reject) => {\n writer.once('open', () => resolve())\n writer.once('error', reject)\n })\n let streamedBytes = 0\n const limiter = new Transform({\n transform(chunk: Buffer, _encoding, callback) {\n const bytes = chunk.length\n if (streamedBytes + bytes > maxBytes) {\n callback(new Error(`Media asset exceeds ${maxBytes} byte limit`))\n return\n }\n if (options.consumeBytes && !options.consumeBytes(bytes)) {\n callback(new Error('Media export total byte limit exceeded'))\n return\n }\n streamedBytes += bytes\n callback(null, chunk)\n },\n })\n try {\n await pipeline(Readable.fromWeb(res.body as import('stream/web').ReadableStream), limiter, writer)\n } catch (error) {\n writer.destroy()\n rmSync(dest, { force: true })\n throw error\n }\n\n const { statSync } = await import('node:fs')\n const sizeBytes = statSync(dest).size\n\n return { savedPath: dest, sizeBytes, mimeType }\n}\n\nexport async function harvestPageMedia(\n html: string,\n pageUrl: string,\n options: MediaExtractOptions = {},\n): Promise<MediaManifest> {\n const types = options.types ?? ['image', 'video', 'audio']\n const typesSet = new Set(types)\n const staticAssets = discoverStaticMedia(html, pageUrl)\n const maxAssets = Math.max(1, Math.min(options.maxAssets ?? 100, 250))\n const allDiscovered = mergeMediaDiscovery(staticAssets, options.rendered, 2_000)\n const discovered = allDiscovered.slice(0, maxAssets)\n const totalFound = new Set([\n ...staticAssets.map(asset => asset.url),\n ...(options.rendered?.assets ?? []).map(asset => asset.url),\n ]).size\n const kept = discovered.flatMap(asset => {\n const type = asset.type ?? typeFromUrl(asset.url)\n return type && typesSet.has(type) ? [{ ...asset, type }] : []\n })\n const filteredCount = totalFound - kept.length\n const retentionLimitReached = allDiscovered.length > maxAssets\n const exhausted = options.rendered?.exhausted === true && !retentionLimitReached\n const stopReason = retentionLimitReached ? 'asset_limit' : options.rendered?.stopReason ?? 'render_unavailable'\n const warnings = retentionLimitReached\n ? [`Media discovery found ${allDiscovered.length} responsive families; only the requested ${maxAssets} records were retained.`]\n : options.rendered\n ? options.rendered.exhausted ? [] : [`Rendered media discovery stopped with ${options.rendered.stopReason}; the inventory may be incomplete.`]\n : ['Rendered media discovery was unavailable; this manifest contains static-source evidence only.']\n\n const baseManifest: Omit<MediaManifest, 'outputDir' | 'assets'> = {\n pageUrl,\n staticFound: staticAssets.length,\n renderedFound: options.rendered?.assets.length ?? 0,\n filteredCount,\n totalFound,\n completeness: exhausted ? 'complete' as const : 'partial' as const,\n exhausted,\n stopReason,\n scrollRounds: options.rendered?.scrollRounds ?? 0,\n warnings,\n artifact: null,\n }\n\n if (options.outputDir === null) {\n return {\n ...baseManifest,\n outputDir: null,\n assets: kept.map((asset, i) => ({\n ...asset,\n mimeType: null, filename: safeFilename(asset.url, i), savedPath: null, sizeBytes: null,\n sha256: null, downloadStatus: 'not_attempted' as const, downloadError: null, inlinePreview: null,\n })),\n }\n }\n\n const domain = (() => { try { return new URL(pageUrl).hostname.replace(/^www\\./, '') } catch { return 'unknown' } })()\n const stamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19)\n const outDir = options.outputDir ?? join(homedir(), 'Downloads', 'mcp-scraper', 'media', `${stamp}-${domain}`)\n mkdirSync(outDir, { recursive: true })\n\n const assets: MediaAsset[] = []\n\n await Promise.allSettled(\n kept.map(async (asset, i) => {\n const filename = safeFilename(asset.url, i)\n try {\n const { savedPath, sizeBytes, mimeType } = await downloadAsset(asset.url, outDir, filename)\n const resolvedType = mimeType ? (typeFromMime(mimeType) ?? asset.type) : asset.type\n assets.push({\n ...asset, type: resolvedType, mimeType, filename: basename(savedPath), savedPath, sizeBytes,\n sha256: null, downloadStatus: 'downloaded', downloadError: null, inlinePreview: null,\n })\n } catch (error) {\n assets.push({\n ...asset, mimeType: null, filename, savedPath: null, sizeBytes: null, sha256: null,\n downloadStatus: 'failed', downloadError: error instanceof Error ? error.message : 'media_download_failed', inlinePreview: null,\n })\n }\n })\n )\n\n assets.sort((a, b) => {\n if (a.savedPath && !b.savedPath) return -1\n if (!a.savedPath && b.savedPath) return 1\n return a.url.localeCompare(b.url)\n })\n\n return { ...baseManifest, outputDir: outDir, assets }\n}\n","import {\n buildPublicErrorEnvelope,\n type PublicChargeStatus,\n type PublicErrorDetails,\n type PublicErrorEnvelope,\n} from '../errors.js'\nimport { isVendorUnavailableError } from './vendor-errors.js'\n\nexport type ExtractionErrorCode =\n | 'vendor_unavailable'\n | 'page_not_found'\n | 'page_forbidden'\n | 'page_rate_limited'\n | 'page_server_error'\n | 'page_http_error'\n | 'bot_check_unresolved'\n | 'page_too_large'\n | 'page_unreachable'\n | 'browser_session_interrupted'\n | 'extraction_failed'\n\nexport interface ExtractionProblem {\n errorCode: ExtractionErrorCode\n httpStatus: number | null\n retryable: boolean\n}\n\nexport interface ExtractionErrorContext {\n chargeStatus?: PublicChargeStatus\n details?: PublicErrorDetails\n}\n\nexport type ExtractionErrorResponse = PublicErrorEnvelope & { error: string }\n\nconst UNREACHABLE_CODES = new Set([\n 'browser_navigation_blocked',\n 'browser_navigation_failed',\n 'browser_timeout',\n 'browser_session_failed',\n 'browser_result_missing',\n])\n\nfunction errorCode(err: unknown): string | undefined {\n return err instanceof Error ? (err as Error & { code?: string }).code : undefined\n}\n\nfunction errorMessage(err: unknown): string {\n return err instanceof Error ? err.message : String(err)\n}\n\nfunction httpStatusFromBrowserHttpCode(value: string | undefined): number | null {\n const match = value?.match(/^browser_http_(\\d+)$/)\n return match ? Number(match[1]) : null\n}\n\nfunction bucketHttpStatus(status: number): ExtractionErrorCode {\n if (status === 404) return 'page_not_found'\n if (status === 403) return 'page_forbidden'\n if (status === 429) return 'page_rate_limited'\n if (status >= 500 && status <= 599) return 'page_server_error'\n return 'page_http_error'\n}\n\nexport function classifyExtractionProblem(err: unknown): ExtractionProblem {\n const message = errorMessage(err)\n const code = errorCode(err)\n\n if (isVendorUnavailableError(err, message)) {\n return { errorCode: 'vendor_unavailable', httpStatus: null, retryable: true }\n }\n\n const httpStatus = httpStatusFromBrowserHttpCode(code) ?? httpStatusFromBrowserHttpCode(message.match(/^(browser_http_\\d+)/)?.[1])\n if (httpStatus != null) {\n return { errorCode: bucketHttpStatus(httpStatus), httpStatus, retryable: httpStatus === 429 || httpStatus >= 500 }\n }\n\n if (code === 'browser_challenge_unresolved' || /^browser_challenge_unresolved/.test(message)) {\n return { errorCode: 'bot_check_unresolved', httpStatus: null, retryable: true }\n }\n\n if (code === 'browser_response_too_large' || /exceeds \\d+ byte limit/i.test(message)) {\n return { errorCode: 'page_too_large', httpStatus: null, retryable: false }\n }\n\n if (/browser (?:has been )?closed|context closed|target page, context or browser has been closed|session closed/i.test(message)) {\n return { errorCode: 'browser_session_interrupted', httpStatus: null, retryable: true }\n }\n\n if (UNREACHABLE_CODES.has(code ?? '') || /timeout|navigation_failed|ERR_/i.test(message)) {\n return { errorCode: 'page_unreachable', httpStatus: null, retryable: true }\n }\n\n return { errorCode: 'extraction_failed', httpStatus: null, retryable: false }\n}\n\nexport function publicExtractionErrorMessage(code: ExtractionErrorCode, httpStatus: number | null): string {\n switch (code) {\n case 'page_not_found':\n return `The page could not be found (HTTP ${httpStatus ?? 404}). It may have been moved or deleted.`\n case 'page_forbidden':\n return `The site refused the request (HTTP ${httpStatus ?? 403}).`\n case 'page_rate_limited':\n return 'The site is rate-limiting requests right now (HTTP 429). Retrying after a short wait usually works.'\n case 'page_server_error':\n return `The page's own server returned an error (HTTP ${httpStatus}). This is on the target site's side, not something a retry here can fix.`\n case 'page_http_error':\n return `The page returned an unexpected HTTP ${httpStatus} response.`\n case 'bot_check_unresolved':\n return 'This site has automated-traffic protection (a bot/CAPTCHA check) that could not be resolved in time. Some sites are simply not extractable this way.'\n case 'page_too_large':\n return 'The page is larger than we can safely process.'\n case 'page_unreachable':\n return 'The page did not respond in time, or the connection could not be completed.'\n case 'browser_session_interrupted':\n return 'The browser session closed before extraction completed. A retry uses a fresh browser session.'\n case 'vendor_unavailable':\n return \"MCP Scraper's internal browser services require MCP Scraper team attention. Servers or IPs are down until this is fixed — this is not caused by your request. Please retry in a few minutes.\"\n case 'extraction_failed':\n return 'The page could not be extracted. Please retry, or contact support if this persists for the same URL.'\n }\n}\n\nexport function extractionProblemResponse(problem: ExtractionProblem, context: ExtractionErrorContext = {}): ExtractionErrorResponse {\n const message = publicExtractionErrorMessage(problem.errorCode, problem.httpStatus)\n const envelope = buildPublicErrorEnvelope({\n errorCode: problem.errorCode,\n retryable: problem.retryable,\n chargeStatus: context.chargeStatus,\n details: {\n ...(context.details ?? {}),\n ...(problem.httpStatus != null ? { http_status: problem.httpStatus } : {}),\n },\n })\n return { ...envelope, message, error: message }\n}\n\nexport function extractionWireStatus(problem: ExtractionProblem): 403 | 404 | 429 | 500 | 502 | 503 {\n if (problem.httpStatus === 404) return 404\n if (problem.httpStatus === 403) return 403\n if (problem.httpStatus === 429) return 429\n if (problem.httpStatus != null && problem.httpStatus >= 500 && problem.httpStatus <= 599) return 500\n if (problem.httpStatus != null) return 502\n if (problem.errorCode === 'vendor_unavailable') return 503\n return 502\n}\n\nconst WAYBACK_CAPTURE_MISSING = 'wayback_capture_missing'\n\nexport function publicizeExtractionFailure(\n failureCode: string | null | undefined,\n failureReason: string | null | undefined,\n): { failureCode: string | null; failureReason: string | null } {\n if (failureCode == null) return { failureCode: null, failureReason: null }\n if (failureCode === WAYBACK_CAPTURE_MISSING) return { failureCode, failureReason: failureReason ?? null }\n const err = new Error(failureReason ?? failureCode) as Error & { code?: string }\n err.code = failureCode\n const problem = classifyExtractionProblem(err)\n return { failureCode: problem.errorCode, failureReason: publicExtractionErrorMessage(problem.errorCode, problem.httpStatus) }\n}\n","import { neon, type NeonQueryFunction } from '@neondatabase/serverless'\nimport { getDb } from './db.js'\n\nexport const COMMONS_EMBED_PROVIDER = 'jina'\n\nlet _vectorSql: NeonQueryFunction<false, false> | null = null\nlet vectorSchemaReady = false\n\nexport function commonsEmbedModel(): string {\n return (process.env.JINA_EMBED_MODEL ?? 'jina-embeddings-v5-omni-small').trim()\n}\n\nexport function commonsEmbedDim(): number {\n return Number((process.env.JINA_EMBED_DIM ?? '1024').trim())\n}\n\nexport function commonsSemanticSearchConfigured(): boolean {\n return Boolean(process.env.JINA_API_KEY?.trim() && process.env.MEMORY_DATABASE_URL?.trim())\n}\n\nfunction vectorSql(): NeonQueryFunction<false, false> {\n if (_vectorSql) return _vectorSql\n const url = process.env.MEMORY_DATABASE_URL?.trim()\n if (!url) throw new Error('MEMORY_DATABASE_URL is not set; Commons semantic search needs the shared Postgres.')\n _vectorSql = neon(url)\n return _vectorSql\n}\n\nexport function resetCommonsVectorSchemaForTesting(): void {\n vectorSchemaReady = false\n _vectorSql = null\n}\n\nexport async function ensureCommonsVectorSchema(): Promise<void> {\n if (vectorSchemaReady) return\n const dimension = commonsEmbedDim()\n await vectorSql().query('CREATE EXTENSION IF NOT EXISTS vector')\n await vectorSql().query(`\n CREATE TABLE IF NOT EXISTS commons_index_vectors (\n document_id TEXT PRIMARY KEY,\n entity_id TEXT NOT NULL,\n document_type TEXT NOT NULL,\n title TEXT NOT NULL,\n embedding vector(${dimension}) NOT NULL,\n model TEXT NOT NULL,\n updated_at TIMESTAMPTZ NOT NULL DEFAULT now()\n )\n `)\n await vectorSql().query('CREATE INDEX IF NOT EXISTS commons_index_vectors_entity ON commons_index_vectors(entity_id)')\n vectorSchemaReady = true\n}\n\nexport async function embedCommonsTexts(texts: string[]): Promise<number[][]> {\n const apiKey = process.env.JINA_API_KEY?.trim()\n if (!apiKey) throw new Error('JINA_API_KEY is not set; Commons semantic search cannot embed.')\n if (!texts.length) return []\n const response = await fetch('https://api.jina.ai/v1/embeddings', {\n method: 'POST',\n headers: { authorization: `Bearer ${apiKey}`, 'content-type': 'application/json' },\n body: JSON.stringify({\n model: commonsEmbedModel(),\n dimensions: commonsEmbedDim(),\n input: texts.map(text => ({ text: text.slice(0, 8_000) })),\n }),\n signal: AbortSignal.timeout(60_000),\n })\n if (!response.ok) {\n throw new Error(`Jina embedding request failed with HTTP ${response.status}: ${(await response.text()).slice(0, 200)}`)\n }\n const payload = await response.json() as { data?: Array<{ embedding: number[] }> }\n const vectors = (payload.data ?? []).map(item => item.embedding)\n if (vectors.length !== texts.length) {\n throw new Error(`Jina returned ${vectors.length} embeddings for ${texts.length} inputs.`)\n }\n return vectors\n}\n\nexport async function embedQueuedCommonsDocuments(limit = 50): Promise<{\n claimed: number\n embedded: number\n failed: number\n remaining: number\n}> {\n if (!commonsSemanticSearchConfigured()) return { claimed: 0, embedded: 0, failed: 0, remaining: 0 }\n await ensureCommonsVectorSchema()\n const bounded = Math.max(1, Math.min(200, Math.floor(limit)))\n const queued = await getDb().execute({\n sql: `SELECT id, entity_id, document_type, title, text FROM commons_index_documents\n WHERE embedding_status IN ('queued', 'failed') ORDER BY updated_at ASC LIMIT ?`,\n args: [bounded],\n })\n const rows = queued.rows as unknown as Array<{ id: string; entity_id: string; document_type: string; title: string; text: string }>\n if (!rows.length) return { claimed: 0, embedded: 0, failed: 0, remaining: await queuedCommonsDocumentCount() }\n\n let embedded = 0\n let failed = 0\n const model = commonsEmbedModel()\n try {\n const vectors = await embedCommonsTexts(rows.map(row => `${row.title}\\n\\n${row.text}`))\n for (const [index, row] of rows.entries()) {\n const literal = `[${vectors[index].join(',')}]`\n await vectorSql().query(\n `INSERT INTO commons_index_vectors (document_id, entity_id, document_type, title, embedding, model, updated_at)\n VALUES ($1, $2, $3, $4, $5::vector, $6, now())\n ON CONFLICT (document_id) DO UPDATE SET entity_id = EXCLUDED.entity_id, document_type = EXCLUDED.document_type,\n title = EXCLUDED.title, embedding = EXCLUDED.embedding, model = EXCLUDED.model, updated_at = now()`,\n [row.id, row.entity_id, row.document_type, row.title, literal, model],\n )\n await getDb().execute({\n sql: `UPDATE commons_index_documents SET embedding_status = 'indexed', embedding_provider = ?, embedding_model = ?,\n vector_ref = ?, indexed_at = ?, error = NULL WHERE id = ?`,\n args: [COMMONS_EMBED_PROVIDER, model, row.id, new Date().toISOString(), row.id],\n })\n embedded += 1\n }\n } catch (error) {\n failed = rows.length - embedded\n const message = (error instanceof Error ? error.message : String(error)).slice(0, 500)\n for (const row of rows.slice(embedded)) {\n await getDb().execute({\n sql: `UPDATE commons_index_documents SET embedding_status = 'failed', error = ? WHERE id = ?`,\n args: [message, row.id],\n }).catch(() => undefined)\n }\n }\n return { claimed: rows.length, embedded, failed, remaining: await queuedCommonsDocumentCount() }\n}\n\nexport async function queuedCommonsDocumentCount(): Promise<number> {\n const result = await getDb().execute(`SELECT COUNT(*) AS n FROM commons_index_documents WHERE embedding_status IN ('queued', 'failed')`)\n return Number((result.rows[0] as unknown as { n: number })?.n ?? 0)\n}\n\nexport async function semanticCommonsEntityScores(query: string, limit = 40): Promise<Map<string, number>> {\n const scores = new Map<string, number>()\n if (!commonsSemanticSearchConfigured() || !query.trim()) return scores\n await ensureCommonsVectorSchema()\n const [vector] = await embedCommonsTexts([query])\n if (!vector) return scores\n const rows = await vectorSql().query(\n `SELECT entity_id, MAX(1 - (embedding <=> $1::vector)) AS score\n FROM commons_index_vectors GROUP BY entity_id ORDER BY score DESC LIMIT $2`,\n [`[${vector.join(',')}]`, Math.max(1, Math.min(100, limit))],\n ) as unknown as Array<{ entity_id: string; score: number }>\n for (const row of rows) scores.set(String(row.entity_id), Number(row.score))\n return scores\n}\n"],"mappings":";;;;;;;;;;;;;;;AAAA,SAAS,mBAAmB,WAAW,cAAc;AACrD,SAAS,eAAe;AACxB,SAAS,MAAM,SAAS,gBAAgB;AACxC,SAAS,gBAAgB;AACzB,SAAS,UAAU,iBAAiB;AAKpC,IAAM,cAAwB;AAAA,EAC5B;AAAA,EAAmB;AAAA,EAAyB;AAAA,EAC5C;AAAA,EAAwB;AAAA,EAAyB;AAAA,EACjD;AAAA,EAAc;AAAA,EACd;AAAA,EAAmB;AAAA,EAAwB;AAAA,EAC3C;AAAA,EAAyB;AAAA,EAA0B;AAAA,EACnD;AAAA,EAAgB;AAAA,EAChB;AAAA,EAAc;AAAA,EAAc;AAAA,EAAkB;AAAA,EAC9C;AAAA,EAAgB;AAAA,EAAe;AAAA,EAAc;AAAA,EAC7C;AAAA,EAAiB;AAAA,EAAgB;AAAA,EAAW;AAAA,EAC5C;AAAA,EAAe;AAAA,EAAc;AAAA,EAAc;AAAA,EAC3C;AAAA,EAAgB;AAAA,EAAa;AAAA,EAAgB;AAAA,EAC7C;AAAA,EAAa;AAAA,EAAe;AAAA,EAAkB;AAAA,EAC9C;AAAA,EAAW;AAAA,EAAc;AAAA,EAAiB;AAAA,EAC1C;AAAA,EAAgB;AAAA,EAAe;AAAA,EAAkB;AAAA,EACjD;AAAA,EAAW;AAAA,EAAe;AAAA,EAAa;AAAA,EACvC;AAAA,EAAa;AAAA,EAAmB;AAAA,EAChC;AAAA,EAAgB;AAAA,EAAiB;AAAA,EAAyB;AAAA,EAC1D;AAAA,EAAS;AAAA,EAAQ;AAAA,EAAY;AAAA,EAAa;AAAA,EAAW;AAAA,EACrD;AAAA,EAAc;AAAA,EAAa;AAAA,EAAiB;AAAA,EAC5C;AAAA,EAAW;AAAA,EAAc;AAAA,EAAa;AACxC;AAEA,IAAM,aAAc,oBAAI,IAAI,CAAC,QAAQ,SAAS,QAAQ,SAAS,QAAQ,SAAS,QAAQ,OAAO,CAAC;AAChG,IAAM,aAAc,oBAAI,IAAI,CAAC,QAAQ,SAAS,QAAQ,QAAQ,QAAQ,QAAQ,MAAM,CAAC;AACrF,IAAM,aAAc,oBAAI,IAAI,CAAC,QAAQ,QAAQ,QAAQ,QAAQ,QAAQ,SAAS,OAAO,CAAC;AAoEtF,SAAS,QAAQ,KAAsB;AACrC,QAAM,QAAQ,IAAI,YAAY;AAC9B,SAAO,YAAY,KAAK,OAAK,MAAM,SAAS,CAAC,CAAC;AAChD;AAEA,SAAS,UAAU,KAAsB;AACvC,SAAO,IAAI,WAAW,OAAO;AAC/B;AAEO,SAAS,YAAY,KAA+B;AACzD,MAAI;AACF,UAAM,MAAM,QAAQ,IAAI,IAAI,GAAG,EAAE,QAAQ,EAAE,YAAY;AACvD,QAAI,WAAW,IAAI,GAAG,EAAG,QAAO;AAChC,QAAI,WAAW,IAAI,GAAG,EAAG,QAAO;AAChC,QAAI,WAAW,IAAI,GAAG,EAAG,QAAO;AAAA,EAClC,QAAQ;AAAA,EAA0B;AAClC,SAAO;AACT;AAEO,SAAS,aAAa,MAAgC;AAC3D,QAAM,QAAQ,KAAK,YAAY;AAC/B,MAAI,MAAM,WAAW,QAAQ,EAAG,QAAO;AACvC,MAAI,MAAM,WAAW,QAAQ,EAAG,QAAO;AACvC,MAAI,MAAM,WAAW,QAAQ,EAAG,QAAO;AACvC,SAAO;AACT;AAEA,SAAS,WAAW,KAAa,MAA6B;AAC5D,MAAI,CAAC,OAAO,UAAU,GAAG,EAAG,QAAO;AACnC,MAAI;AAAE,WAAO,IAAI,IAAI,KAAK,IAAI,EAAE;AAAA,EAAK,QAAQ;AAAE,WAAO;AAAA,EAAK;AAC7D;AAEA,SAAS,aAAa,KAAa,OAAuB;AACxD,MAAI;AACF,UAAM,IAAI,IAAI,IAAI,GAAG;AACrB,UAAM,OAAO,SAAS,EAAE,QAAQ,EAAE,QAAQ,oBAAoB,GAAG,EAAE,MAAM,GAAG,EAAE;AAC9E,WAAO,QAAQ,SAAS,KAAK;AAAA,EAC/B,QAAQ;AACN,WAAO,SAAS,KAAK;AAAA,EACvB;AACF;AAEA,SAAS,YAAY,OAAkC,KAA4B;AACjF,QAAM,aAAa,OAAO,QAAQ,0BAA0B,GAAG,EAAE,QAAQ,QAAQ,GAAG,EAAE,KAAK;AAC3F,SAAO,aAAa,WAAW,MAAM,GAAG,GAAG,IAAI;AACjD;AAEA,SAAS,kBAAkB,QAA0D;AACnF,QAAM,QAAQ,OAAO,MAAM,yCAAyC;AACpE,SAAO,QAAQ,EAAE,OAAO,OAAO,MAAM,CAAC,CAAC,GAAG,QAAQ,OAAO,MAAM,CAAC,CAAC,EAAE,IAAI;AACzE;AAEO,SAAS,eAAe,QAAwB;AACrD,MAAI;AACF,UAAM,MAAM,IAAI,IAAI,MAAM;AAC1B,QAAI,OAAO;AACX,QAAI,WAAW,IAAI,SAAS,QAAQ,4DAA4D,EAAE;AAClG,eAAW,OAAO,CAAC,KAAK,SAAS,KAAK,UAAU,QAAQ,EAAG,KAAI,aAAa,OAAO,GAAG;AACtF,QAAI,aAAa,KAAK;AACtB,WAAO,IAAI;AAAA,EACb,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEA,SAAS,eAAe,WAA6C;AACnE,QAAM,QAAQ,UAAU,SAAS,MAAM,UAAU,UAAU;AAC3D,QAAM,cAAc,UAAU,iBAAiB,KAAK,YAAU,OAAO,SAAS,UAAU,CAAC,IAAI,OAAoB;AACjH,QAAM,gBAAgB,oEAAoE,KAAK,UAAU,GAAG,IAAI,IAAI;AACpH,SAAO,cAAc,gBAAgB;AACvC;AAEA,SAAS,eAAe,QAA+C,UAA0C;AAC/G,QAAM,WAAW,OAAO,IAAI,SAAS,GAAG;AACxC,MAAI,CAAC,UAAU;AACb,WAAO,IAAI,SAAS,KAAK,QAAQ;AACjC;AAAA,EACF;AACA,WAAS,mBAAmB,CAAC,GAAG,oBAAI,IAAI,CAAC,GAAG,SAAS,kBAAkB,GAAG,SAAS,gBAAgB,CAAC,CAAC;AACrG,WAAS,WAAW,CAAC,GAAG,oBAAI,IAAI,CAAC,GAAG,SAAS,UAAU,GAAG,SAAS,QAAQ,CAAC,CAAC;AAC7E,WAAS,WAAW,CAAC,GAAG,oBAAI,IAAI,CAAC,GAAG,SAAS,UAAU,GAAG,SAAS,QAAQ,CAAC,CAAC;AAC7E,MAAI,CAAC,SAAS,QAAQ,SAAS,KAAM,UAAS,OAAO,SAAS;AAC9D,OAAK,SAAS,SAAS,MAAM,SAAS,UAAU,MAAM,SAAS,SAAS,MAAM,SAAS,UAAU,IAAI;AACnG,aAAS,QAAQ,SAAS;AAC1B,aAAS,SAAS,SAAS;AAAA,EAC7B;AACF;AAEO,SAAS,oBAAoB,MAA6B,SAA6C;AAC5G,QAAM,QAAQ,oBAAI,IAAsC;AACxD,QAAM,MAAM,CACV,KACA,QACA,eAAiC,MACjC,KACA,SACA,eACG;AACH,UAAM,WAAW,MAAM,WAAW,IAAI,KAAK,EAAE,QAAQ,UAAU,GAAG,GAAG,OAAO,IAAI;AAChF,QAAI,CAAC,YAAY,QAAQ,QAAQ,EAAG;AACpC,UAAM,qBAAqB,cAAc,kBAAkB,QAAQ;AACnE,mBAAe,OAAO;AAAA,MACpB,KAAK;AAAA,MACL,MAAM,gBAAgB,YAAY,QAAQ;AAAA,MAC1C,kBAAkB,CAAC,MAAM;AAAA,MACzB,UAAU,MAAM,CAAC,GAAG,IAAI,CAAC;AAAA,MACzB,UAAU,UAAU,CAAC,OAAO,IAAI,CAAC;AAAA,MACjC,OAAO,oBAAoB,SAAS;AAAA,MACpC,QAAQ,oBAAoB,UAAU;AAAA,IACxC,CAAC;AAAA,EACH;AACA,QAAM,aAAa,CAAC,OAA2B,QAAgB,MAAiB,QAAwB;AACtG,eAAW,SAAS,SAAS,IAAI,MAAM,GAAG,GAAG;AAC3C,YAAM,CAAC,KAAK,UAAU,IAAI,KAAK,KAAK,EAAE,MAAM,KAAK;AACjD,UAAI,CAAC,IAAK;AACV,YAAM,QAAQ,YAAY,MAAM,UAAU,IAAI,CAAC;AAC/C,UAAI,KAAK,QAAQ,MAAM,KAAK,MAAM,QAAQ,EAAE,OAAO,OAAO,KAAK,GAAG,QAAQ,EAAE,IAAI,IAAI;AAAA,IACtF;AAAA,EACF;AACA,QAAM,OAAO,CAACA,WAAwB,SAAS,OAAO;AACpD,UAAM,IAAIA,UAAS;AACnB,eAAW,SAASA,UAAS,QAAQ;AACnC,YAAM,QAAQ,MAAM;AACpB,YAAM,MAAM,YAAY,MAAM,KAAK,GAAG;AACtC,YAAM,aAAa,MAAM,SAAS,MAAM,SAAS,EAAE,OAAO,MAAM,OAAO,QAAQ,MAAM,OAAO,IAAI;AAChG,iBAAW,QAAQ,CAAC,OAAO,YAAY,iBAAiB,iBAAiB,WAAW,mBAAmB,eAAe,gBAAgB,WAAW,GAAG;AAClJ,cAAM,QAAQ,MAAM,IAAI;AACxB,YAAI,OAAO,UAAU,IAAI,GAAG,MAAM,IAAI,SAAS,KAAK,MAAM,UAAU;AAAA,MACtE;AACA,iBAAW,MAAM,QAAQ,gBAAgB,MAAM,IAAI,SAAS,GAAG;AAC/D,iBAAW,MAAM,aAAa,GAAG,qBAAqB,MAAM,IAAI,SAAS,GAAG;AAAA,IAC9E;AACA,MAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,YAAY;AACpC,YAAM,OAAO,EAAE,OAAO;AACtB,YAAM,WAAW,KAAK,KAAK,MAAM,KAAK;AACtC,YAAM,OAAkB,SAAS,WAAW,QAAQ,IAAI,UAAU,SAAS,WAAW,QAAQ,IAAI,UAAU;AAC5G,UAAI,KAAK,KAAK,KAAK,GAAG,gBAAgB,MAAM,IAAI,IAAI;AACpD,iBAAW,KAAK,KAAK,QAAQ,GAAG,uBAAuB,MAAM,IAAI,IAAI;AAAA,IACvE,CAAC;AACD,MAAE,aAAa,EAAE,KAAK,CAAC,QAAQ,YAAY;AACzC,YAAM,OAAO,EAAE,OAAO;AACtB,YAAM,OAAO,QAAQ,QAAQ,YAAY;AACzC,UAAI,KAAK,KAAK,KAAK,GAAG,UAAU,IAAI,GAAG,MAAM,IAAI,IAAI;AACrD,UAAI,SAAS,QAAS,KAAI,KAAK,KAAK,QAAQ,GAAG,sBAAsB,MAAM,IAAI,OAAO;AAAA,IACxF,CAAC;AACD,MAAE,MAAM,EAAE,KAAK,CAAC,QAAQ,YAAY;AAClC,YAAM,OAAO,EAAE,OAAO;AACtB,YAAM,QAAQ,KAAK,KAAK,UAAU,KAAK,KAAK,KAAK,MAAM,KAAK,IAAI,YAAY;AAC5E,UAAI,SAAS,cAAc,SAAS,kBAAkB,SAAS,mBAAmB,SAAS,qBAAqB;AAC9G,YAAI,KAAK,KAAK,SAAS,GAAG,UAAU,IAAI,GAAG,MAAM,IAAI,OAAO;AAAA,MAC9D;AAAA,IACF,CAAC;AACD,MAAE,YAAY,EAAE,KAAK,CAAC,QAAQ,YAAY;AACxC,YAAM,OAAO,EAAE,OAAO;AACtB,YAAM,OAAO,KAAK,KAAK,KAAK,KAAK,IAAI,YAAY,EAAE,MAAM,KAAK;AAC9D,UAAI,IAAI,KAAK,WAAS,UAAU,UAAU,UAAU,kBAAkB,GAAG;AACvE,YAAI,KAAK,KAAK,MAAM,GAAG,cAAc,MAAM,IAAI,OAAO;AAAA,MACxD;AAAA,IACF,CAAC;AACD,MAAE,WAAW,EAAE,KAAK,CAAC,QAAQ,YAAY;AACvC,YAAM,OAAO,EAAE,OAAO;AACtB,UAAI,KAAK,KAAK,MAAM,KAAK,KAAK,KAAK,YAAY,GAAG,mBAAmB,MAAM,IAAI,OAAO;AAAA,IACxF,CAAC;AACD,MAAE,SAAS,EAAE,IAAI,KAAK,EAAE,KAAK,CAAC,QAAQ,YAAY;AAChD,YAAM,OAAO,EAAE,OAAO;AACtB,YAAM,OAAO,KAAK,KAAK,MAAM;AAC7B,UAAI,CAAC,KAAM;AACX,UACE,wEAAwE,KAAK,GAAG,KAAK,KAAK,OAAO,KAAK,EAAE,IAAI,KAAK,KAAK,8BAA8B,KAAK,EAAE,EAAE,KAC1J,+CAA+C,KAAK,IAAI,GAC3D;AACA,cAAM,MAAM,YAAY,KAAK,KAAK,KAAK,EAAE,MAAM,EAAE,KAAK,KAAK,GAAG,GAAG;AACjE,YAAI,MAAM,yBAAyB,MAAM,IAAI,SAAS,GAAG;AAAA,MAC3D;AAAA,IACF,CAAC;AACD,UAAM,aAAa,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,aAAW,EAAE,OAAO,EAAE,KAAK,CAAC;AACxE,MAAE,SAAS,EAAE,KAAK,CAAC,QAAQ,YAAY;AAAE,iBAAW,KAAK,EAAE,OAAO,EAAE,KAAK,OAAO,KAAK,EAAE;AAAA,IAAE,CAAC;AAC1F,eAAW,SAAS,WAAW,KAAK,IAAI,EAAE,SAAS,0IAA0I,GAAG;AAC9L,UAAI,MAAM,CAAC,GAAG,iBAAiB,MAAM,IAAI,OAAO;AAAA,IAClD;AACA,eAAW,SAASA,UAAS,WAAW,SAAS,sGAAsG,GAAG;AACxJ,UAAI,MAAM,CAAC,GAAG,kBAAkB,MAAM,EAAE;AAAA,IAC1C;AAAA,EACF;AAEA,QAAM,WAAW,OAAO,SAAS,WAAW,iBAAiB,IAAI,IAAI;AACrE,OAAK,QAAQ;AACb,WAAS,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,YAAY;AAC7C,UAAM,SAAS,SAAS,EAAE,OAAO,EAAE,KAAK;AACxC,UAAM,YAAY,OACf,QAAQ,eAAe,GAAG,EAAE,QAAQ,eAAe,GAAG,EAAE,QAAQ,aAAa,GAAG,EAChF,QAAQ,WAAW,WAAS,MAAM,MAAM,CAAC,CAAC,EAAE,QAAQ,SAAS,GAAG;AACnE,QAAI,cAAc,UAAU,iCAAiC,KAAK,SAAS,GAAG;AAC5E,WAAK,iBAAiB,SAAS,GAAG,iBAAiB;AAAA,IACrD;AAAA,EACF,CAAC;AACD,SAAO,CAAC,GAAG,MAAM,OAAO,CAAC;AAC3B;AAEO,SAAS,oBACd,cACA,UACA,YAAY,KAC8C;AAC1D,QAAM,QAAQ,oBAAI,IAAsC;AACxD,aAAW,SAAS,aAAc,gBAAe,OAAO,KAAK;AAC7D,aAAW,SAAS,UAAU,UAAU,CAAC,EAAG,gBAAe,OAAO,EAAE,GAAG,OAAO,MAAM,MAAM,KAAK,CAAC;AAEhG,QAAM,WAAW,oBAAI,IAAwC;AAC7D,aAAW,SAAS,MAAM,OAAO,GAAG;AAClC,UAAM,MAAM,eAAe,MAAM,GAAG;AACpC,UAAM,SAAS,SAAS,IAAI,GAAG,KAAK,CAAC;AACrC,WAAO,KAAK,KAAK;AACjB,aAAS,IAAI,KAAK,MAAM;AAAA,EAC1B;AACA,SAAO,CAAC,GAAG,SAAS,OAAO,CAAC,EACzB,IAAI,YAAU;AACb,UAAM,UAAU,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,eAAe,CAAC,IAAI,eAAe,CAAC,KAAK,EAAE,IAAI,cAAc,EAAE,GAAG,CAAC;AAC9G,UAAM,YAAY,EAAE,GAAG,QAAQ,CAAC,EAAE;AAClC,cAAU,mBAAmB,CAAC,GAAG,IAAI,IAAI,OAAO,QAAQ,WAAS,MAAM,gBAAgB,CAAC,CAAC;AACzF,cAAU,WAAW,CAAC,GAAG,IAAI,IAAI,OAAO,QAAQ,WAAS,MAAM,QAAQ,CAAC,CAAC;AACzE,cAAU,WAAW,CAAC,GAAG,IAAI,IAAI,OAAO,QAAQ,WAAS,MAAM,QAAQ,CAAC,CAAC;AACzE,WAAO,EAAE,GAAG,WAAW,UAAU,QAAQ,MAAM,CAAC,EAAE,IAAI,WAAS,MAAM,GAAG,EAAE;AAAA,EAC5E,CAAC,EACA,KAAK,CAAC,GAAG,MAAM,eAAe,CAAC,IAAI,eAAe,CAAC,KAAK,EAAE,IAAI,cAAc,EAAE,GAAG,CAAC,EAClF,MAAM,GAAG,KAAK,IAAI,GAAG,SAAS,CAAC;AACpC;AAMA,eAAsB,cACpB,KACA,SACA,UACA,UAII,CAAC,GACuE;AAC5E,QAAM,WAAW,KAAK,IAAI,GAAG,KAAK,IAAI,QAAQ,YAAY,KAAK,OAAO,MAAM,MAAM,OAAO,IAAI,CAAC;AAC9F,MAAI,SAAS;AACb,MAAI,MAAuB;AAC3B,WAAS,YAAY,GAAG,aAAa,GAAG,aAAa;AACnD,UAAM,UAAU,MAAM,sBAAsB,QAAQ,EAAE,OAAO,YAAY,CAAC;AAC1E,QAAI,QAAQ,SAAS,CAAC,QAAQ,OAAQ,OAAM,IAAI,MAAM,QAAQ,SAAS,wBAAwB;AAC/F,UAAM,MAAM,MAAM,QAAQ,OAAO,MAAM;AAAA,MACrC,QAAQ,YAAY,QAAQ,IAAM;AAAA,MAClC,UAAU;AAAA,IACZ,CAAC;AACD,QAAI,IAAI,UAAU,OAAO,IAAI,SAAS,KAAK;AACzC,YAAM,WAAW,IAAI,QAAQ,IAAI,UAAU;AAC3C,UAAI,CAAC,SAAU,OAAM,IAAI,MAAM,QAAQ,IAAI,MAAM,oCAAoC;AACrF,eAAS,IAAI,IAAI,UAAU,QAAQ,OAAO,IAAI,EAAE;AAChD,YAAM;AACN;AAAA,IACF;AACA;AAAA,EACF;AACA,MAAI,CAAC,IAAK,OAAM,IAAI,MAAM,wCAAwC;AAClE,MAAI,CAAC,IAAI,GAAI,OAAM,IAAI,MAAM,QAAQ,IAAI,MAAM,EAAE;AACjD,MAAI,CAAC,IAAI,KAAM,OAAM,IAAI,MAAM,qBAAqB;AAEpD,QAAM,WAAW,IAAI,QAAQ,IAAI,cAAc,GAAG,MAAM,GAAG,EAAE,CAAC,EAAE,KAAK,KAAK;AAC1E,MAAI,QAAQ,iBAAiB,CAAC,YAAY,aAAa,QAAQ,MAAM,QAAQ,eAAe;AAC1F,UAAM,IAAI,MAAM,YAAY,QAAQ,YAAY,yBAAyB,YAAY,iBAAiB,EAAE;AAAA,EAC1G;AACA,QAAM,gBAAgB,OAAO,IAAI,QAAQ,IAAI,gBAAgB,KAAK,CAAC;AACnE,MAAI,OAAO,SAAS,aAAa,KAAK,gBAAgB,UAAU;AAC9D,UAAM,IAAI,MAAM,uBAAuB,QAAQ,aAAa;AAAA,EAC9D;AAEA,MAAI,OAAO,KAAK,SAAS,QAAQ;AACjC,MAAI,YAAY,CAAC,QAAQ,QAAQ,GAAG;AAClC,UAAM,UAAkC;AAAA,MACtC,cAAc;AAAA,MAAQ,aAAa;AAAA,MAAQ,cAAc;AAAA,MACzD,aAAa;AAAA,MAAQ,iBAAiB;AAAA,MAAQ,cAAc;AAAA,MAC5D,aAAa;AAAA,MAAQ,cAAc;AAAA,MACnC,cAAc;AAAA,MAAQ,aAAa;AAAA,MAAQ,aAAa;AAAA,IAC1D;AACA,UAAM,MAAM,QAAQ,QAAQ;AAC5B,QAAI,IAAK,QAAO,OAAO;AAAA,EACzB;AAEA,QAAM,SAAS,kBAAkB,IAAI;AACrC,QAAM,IAAI,QAAc,CAAC,SAAS,WAAW;AAC3C,WAAO,KAAK,QAAQ,MAAM,QAAQ,CAAC;AACnC,WAAO,KAAK,SAAS,MAAM;AAAA,EAC7B,CAAC;AACD,MAAI,gBAAgB;AACpB,QAAM,UAAU,IAAI,UAAU;AAAA,IAC5B,UAAU,OAAe,WAAW,UAAU;AAC5C,YAAM,QAAQ,MAAM;AACpB,UAAI,gBAAgB,QAAQ,UAAU;AACpC,iBAAS,IAAI,MAAM,uBAAuB,QAAQ,aAAa,CAAC;AAChE;AAAA,MACF;AACA,UAAI,QAAQ,gBAAgB,CAAC,QAAQ,aAAa,KAAK,GAAG;AACxD,iBAAS,IAAI,MAAM,wCAAwC,CAAC;AAC5D;AAAA,MACF;AACA,uBAAiB;AACjB,eAAS,MAAM,KAAK;AAAA,IACtB;AAAA,EACF,CAAC;AACD,MAAI;AACF,UAAM,SAAS,SAAS,QAAQ,IAAI,IAA2C,GAAG,SAAS,MAAM;AAAA,EACnG,SAAS,OAAO;AACd,WAAO,QAAQ;AACf,WAAO,MAAM,EAAE,OAAO,KAAK,CAAC;AAC5B,UAAM;AAAA,EACR;AAEA,QAAM,EAAE,SAAS,IAAI,MAAM,OAAO,IAAS;AAC3C,QAAM,YAAY,SAAS,IAAI,EAAE;AAEjC,SAAO,EAAE,WAAW,MAAM,WAAW,SAAS;AAChD;AAEA,eAAsB,iBACpB,MACA,SACA,UAA+B,CAAC,GACR;AACxB,QAAM,QAAW,QAAQ,SAAS,CAAC,SAAS,SAAS,OAAO;AAC5D,QAAM,WAAW,IAAI,IAAI,KAAK;AAC9B,QAAM,eAAe,oBAAoB,MAAM,OAAO;AACtD,QAAM,YAAY,KAAK,IAAI,GAAG,KAAK,IAAI,QAAQ,aAAa,KAAK,GAAG,CAAC;AACrE,QAAM,gBAAgB,oBAAoB,cAAc,QAAQ,UAAU,GAAK;AAC/E,QAAM,aAAa,cAAc,MAAM,GAAG,SAAS;AACnD,QAAM,cAAa,oBAAI,IAAI;AAAA,IACzB,GAAG,aAAa,IAAI,WAAS,MAAM,GAAG;AAAA,IACtC,IAAI,QAAQ,UAAU,UAAU,CAAC,GAAG,IAAI,WAAS,MAAM,GAAG;AAAA,EAC5D,CAAC,GAAE;AACH,QAAM,OAAO,WAAW,QAAQ,WAAS;AACvC,UAAM,OAAO,MAAM,QAAQ,YAAY,MAAM,GAAG;AAChD,WAAO,QAAQ,SAAS,IAAI,IAAI,IAAI,CAAC,EAAE,GAAG,OAAO,KAAK,CAAC,IAAI,CAAC;AAAA,EAC9D,CAAC;AACD,QAAM,gBAAgB,aAAa,KAAK;AACxC,QAAM,wBAAwB,cAAc,SAAS;AACrD,QAAM,YAAY,QAAQ,UAAU,cAAc,QAAQ,CAAC;AAC3D,QAAM,aAAa,wBAAwB,gBAAgB,QAAQ,UAAU,cAAc;AAC3F,QAAM,WAAW,wBACb,CAAC,yBAAyB,cAAc,MAAM,4CAA4C,SAAS,yBAAyB,IAC5H,QAAQ,WACR,QAAQ,SAAS,YAAY,CAAC,IAAI,CAAC,yCAAyC,QAAQ,SAAS,UAAU,oCAAoC,IAC3I,CAAC,+FAA+F;AAEpG,QAAM,eAA4D;AAAA,IAChE;AAAA,IACA,aAAa,aAAa;AAAA,IAC1B,eAAe,QAAQ,UAAU,OAAO,UAAU;AAAA,IAClD;AAAA,IACA;AAAA,IACA,cAAc,YAAY,aAAsB;AAAA,IAChD;AAAA,IACA;AAAA,IACA,cAAc,QAAQ,UAAU,gBAAgB;AAAA,IAChD;AAAA,IACA,UAAU;AAAA,EACZ;AAEA,MAAI,QAAQ,cAAc,MAAM;AAC9B,WAAO;AAAA,MACL,GAAG;AAAA,MACH,WAAW;AAAA,MACX,QAAQ,KAAK,IAAI,CAAC,OAAO,OAAO;AAAA,QAC9B,GAAG;AAAA,QACH,UAAU;AAAA,QAAM,UAAU,aAAa,MAAM,KAAK,CAAC;AAAA,QAAG,WAAW;AAAA,QAAM,WAAW;AAAA,QAClF,QAAQ;AAAA,QAAM,gBAAgB;AAAA,QAA0B,eAAe;AAAA,QAAM,eAAe;AAAA,MAC9F,EAAE;AAAA,IACJ;AAAA,EACF;AAEA,QAAM,UAAW,MAAM;AAAE,QAAI;AAAE,aAAO,IAAI,IAAI,OAAO,EAAE,SAAS,QAAQ,UAAU,EAAE;AAAA,IAAE,QAAQ;AAAE,aAAO;AAAA,IAAU;AAAA,EAAE,GAAG;AACtH,QAAM,SAAU,oBAAI,KAAK,GAAE,YAAY,EAAE,QAAQ,SAAS,GAAG,EAAE,MAAM,GAAG,EAAE;AAC1E,QAAM,SAAU,QAAQ,aAAa,KAAK,QAAQ,GAAG,aAAa,eAAe,SAAS,GAAG,KAAK,IAAI,MAAM,EAAE;AAC9G,YAAU,QAAQ,EAAE,WAAW,KAAK,CAAC;AAErC,QAAM,SAAuB,CAAC;AAE9B,QAAM,QAAQ;AAAA,IACZ,KAAK,IAAI,OAAO,OAAO,MAAM;AAC3B,YAAM,WAAW,aAAa,MAAM,KAAK,CAAC;AAC1C,UAAI;AACF,cAAM,EAAE,WAAW,WAAW,SAAS,IAAI,MAAM,cAAc,MAAM,KAAK,QAAQ,QAAQ;AAC1F,cAAM,eAAe,WAAY,aAAa,QAAQ,KAAK,MAAM,OAAQ,MAAM;AAC/E,eAAO,KAAK;AAAA,UACV,GAAG;AAAA,UAAO,MAAM;AAAA,UAAc;AAAA,UAAU,UAAU,SAAS,SAAS;AAAA,UAAG;AAAA,UAAW;AAAA,UAClF,QAAQ;AAAA,UAAM,gBAAgB;AAAA,UAAc,eAAe;AAAA,UAAM,eAAe;AAAA,QAClF,CAAC;AAAA,MACH,SAAS,OAAO;AACd,eAAO,KAAK;AAAA,UACV,GAAG;AAAA,UAAO,UAAU;AAAA,UAAM;AAAA,UAAU,WAAW;AAAA,UAAM,WAAW;AAAA,UAAM,QAAQ;AAAA,UAC9E,gBAAgB;AAAA,UAAU,eAAe,iBAAiB,QAAQ,MAAM,UAAU;AAAA,UAAyB,eAAe;AAAA,QAC5H,CAAC;AAAA,MACH;AAAA,IACF,CAAC;AAAA,EACH;AAEA,SAAO,KAAK,CAAC,GAAG,MAAM;AACpB,QAAI,EAAE,aAAa,CAAC,EAAE,UAAW,QAAO;AACxC,QAAI,CAAC,EAAE,aAAa,EAAE,UAAW,QAAO;AACxC,WAAO,EAAE,IAAI,cAAc,EAAE,GAAG;AAAA,EAClC,CAAC;AAED,SAAO,EAAE,GAAG,cAAc,WAAW,QAAQ,OAAO;AACtD;;;AC7dA,IAAM,oBAAoB,oBAAI,IAAI;AAAA,EAChC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAED,SAAS,UAAU,KAAkC;AACnD,SAAO,eAAe,QAAS,IAAkC,OAAO;AAC1E;AAEA,SAAS,aAAa,KAAsB;AAC1C,SAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AACxD;AAEA,SAAS,8BAA8B,OAA0C;AAC/E,QAAM,QAAQ,OAAO,MAAM,sBAAsB;AACjD,SAAO,QAAQ,OAAO,MAAM,CAAC,CAAC,IAAI;AACpC;AAEA,SAAS,iBAAiB,QAAqC;AAC7D,MAAI,WAAW,IAAK,QAAO;AAC3B,MAAI,WAAW,IAAK,QAAO;AAC3B,MAAI,WAAW,IAAK,QAAO;AAC3B,MAAI,UAAU,OAAO,UAAU,IAAK,QAAO;AAC3C,SAAO;AACT;AAEO,SAAS,0BAA0B,KAAiC;AACzE,QAAM,UAAU,aAAa,GAAG;AAChC,QAAM,OAAO,UAAU,GAAG;AAE1B,MAAI,yBAAyB,KAAK,OAAO,GAAG;AAC1C,WAAO,EAAE,WAAW,sBAAsB,YAAY,MAAM,WAAW,KAAK;AAAA,EAC9E;AAEA,QAAM,aAAa,8BAA8B,IAAI,KAAK,8BAA8B,QAAQ,MAAM,qBAAqB,IAAI,CAAC,CAAC;AACjI,MAAI,cAAc,MAAM;AACtB,WAAO,EAAE,WAAW,iBAAiB,UAAU,GAAG,YAAY,WAAW,eAAe,OAAO,cAAc,IAAI;AAAA,EACnH;AAEA,MAAI,SAAS,kCAAkC,gCAAgC,KAAK,OAAO,GAAG;AAC5F,WAAO,EAAE,WAAW,wBAAwB,YAAY,MAAM,WAAW,KAAK;AAAA,EAChF;AAEA,MAAI,SAAS,gCAAgC,0BAA0B,KAAK,OAAO,GAAG;AACpF,WAAO,EAAE,WAAW,kBAAkB,YAAY,MAAM,WAAW,MAAM;AAAA,EAC3E;AAEA,MAAI,8GAA8G,KAAK,OAAO,GAAG;AAC/H,WAAO,EAAE,WAAW,+BAA+B,YAAY,MAAM,WAAW,KAAK;AAAA,EACvF;AAEA,MAAI,kBAAkB,IAAI,QAAQ,EAAE,KAAK,kCAAkC,KAAK,OAAO,GAAG;AACxF,WAAO,EAAE,WAAW,oBAAoB,YAAY,MAAM,WAAW,KAAK;AAAA,EAC5E;AAEA,SAAO,EAAE,WAAW,qBAAqB,YAAY,MAAM,WAAW,MAAM;AAC9E;AAEO,SAAS,6BAA6B,MAA2B,YAAmC;AACzG,UAAQ,MAAM;AAAA,IACZ,KAAK;AACH,aAAO,qCAAqC,cAAc,GAAG;AAAA,IAC/D,KAAK;AACH,aAAO,sCAAsC,cAAc,GAAG;AAAA,IAChE,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO,iDAAiD,UAAU;AAAA,IACpE,KAAK;AACH,aAAO,wCAAwC,UAAU;AAAA,IAC3D,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,EACX;AACF;AAEO,SAAS,0BAA0B,SAA4B,UAAkC,CAAC,GAA4B;AACnI,QAAM,UAAU,6BAA6B,QAAQ,WAAW,QAAQ,UAAU;AAClF,QAAM,WAAW,yBAAyB;AAAA,IACxC,WAAW,QAAQ;AAAA,IACnB,WAAW,QAAQ;AAAA,IACnB,cAAc,QAAQ;AAAA,IACtB,SAAS;AAAA,MACP,GAAI,QAAQ,WAAW,CAAC;AAAA,MACxB,GAAI,QAAQ,cAAc,OAAO,EAAE,aAAa,QAAQ,WAAW,IAAI,CAAC;AAAA,IAC1E;AAAA,EACF,CAAC;AACD,SAAO,EAAE,GAAG,UAAU,SAAS,OAAO,QAAQ;AAChD;AAEO,SAAS,qBAAqB,SAA+D;AAClG,MAAI,QAAQ,eAAe,IAAK,QAAO;AACvC,MAAI,QAAQ,eAAe,IAAK,QAAO;AACvC,MAAI,QAAQ,eAAe,IAAK,QAAO;AACvC,MAAI,QAAQ,cAAc,QAAQ,QAAQ,cAAc,OAAO,QAAQ,cAAc,IAAK,QAAO;AACjG,MAAI,QAAQ,cAAc,KAAM,QAAO;AACvC,MAAI,QAAQ,cAAc,qBAAsB,QAAO;AACvD,SAAO;AACT;AAEA,IAAM,0BAA0B;AAEzB,SAAS,2BACd,aACA,eAC8D;AAC9D,MAAI,eAAe,KAAM,QAAO,EAAE,aAAa,MAAM,eAAe,KAAK;AACzE,MAAI,gBAAgB,wBAAyB,QAAO,EAAE,aAAa,eAAe,iBAAiB,KAAK;AACxG,QAAM,MAAM,IAAI,MAAM,iBAAiB,WAAW;AAClD,MAAI,OAAO;AACX,QAAM,UAAU,0BAA0B,GAAG;AAC7C,SAAO,EAAE,aAAa,QAAQ,WAAW,eAAe,6BAA6B,QAAQ,WAAW,QAAQ,UAAU,EAAE;AAC9H;;;AC9JA,SAAS,YAAoC;AAGtC,IAAM,yBAAyB;AAEtC,IAAI,aAAqD;AACzD,IAAI,oBAAoB;AAEjB,SAAS,oBAA4B;AAC1C,UAAQ,QAAQ,IAAI,oBAAoB,iCAAiC,KAAK;AAChF;AAEO,SAAS,kBAA0B;AACxC,SAAO,QAAQ,QAAQ,IAAI,kBAAkB,QAAQ,KAAK,CAAC;AAC7D;AAEO,SAAS,kCAA2C;AACzD,SAAO,QAAQ,QAAQ,IAAI,cAAc,KAAK,KAAK,QAAQ,IAAI,qBAAqB,KAAK,CAAC;AAC5F;AAEA,SAAS,YAA6C;AACpD,MAAI,WAAY,QAAO;AACvB,QAAM,MAAM,QAAQ,IAAI,qBAAqB,KAAK;AAClD,MAAI,CAAC,IAAK,OAAM,IAAI,MAAM,oFAAoF;AAC9G,eAAa,KAAK,GAAG;AACrB,SAAO;AACT;AAOA,eAAsB,4BAA2C;AAC/D,MAAI,kBAAmB;AACvB,QAAM,YAAY,gBAAgB;AAClC,QAAM,UAAU,EAAE,MAAM,uCAAuC;AAC/D,QAAM,UAAU,EAAE,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,6BAMG,SAAS;AAAA;AAAA;AAAA;AAAA,GAInC;AACD,QAAM,UAAU,EAAE,MAAM,6FAA6F;AACrH,sBAAoB;AACtB;AAEA,eAAsB,kBAAkB,OAAsC;AAC5E,QAAM,SAAS,QAAQ,IAAI,cAAc,KAAK;AAC9C,MAAI,CAAC,OAAQ,OAAM,IAAI,MAAM,gEAAgE;AAC7F,MAAI,CAAC,MAAM,OAAQ,QAAO,CAAC;AAC3B,QAAM,WAAW,MAAM,MAAM,qCAAqC;AAAA,IAChE,QAAQ;AAAA,IACR,SAAS,EAAE,eAAe,UAAU,MAAM,IAAI,gBAAgB,mBAAmB;AAAA,IACjF,MAAM,KAAK,UAAU;AAAA,MACnB,OAAO,kBAAkB;AAAA,MACzB,YAAY,gBAAgB;AAAA,MAC5B,OAAO,MAAM,IAAI,WAAS,EAAE,MAAM,KAAK,MAAM,GAAG,GAAK,EAAE,EAAE;AAAA,IAC3D,CAAC;AAAA,IACD,QAAQ,YAAY,QAAQ,GAAM;AAAA,EACpC,CAAC;AACD,MAAI,CAAC,SAAS,IAAI;AAChB,UAAM,IAAI,MAAM,2CAA2C,SAAS,MAAM,MAAM,MAAM,SAAS,KAAK,GAAG,MAAM,GAAG,GAAG,CAAC,EAAE;AAAA,EACxH;AACA,QAAM,UAAU,MAAM,SAAS,KAAK;AACpC,QAAM,WAAW,QAAQ,QAAQ,CAAC,GAAG,IAAI,UAAQ,KAAK,SAAS;AAC/D,MAAI,QAAQ,WAAW,MAAM,QAAQ;AACnC,UAAM,IAAI,MAAM,iBAAiB,QAAQ,MAAM,mBAAmB,MAAM,MAAM,UAAU;AAAA,EAC1F;AACA,SAAO;AACT;AAEA,eAAsB,4BAA4B,QAAQ,IAKvD;AACD,MAAI,CAAC,gCAAgC,EAAG,QAAO,EAAE,SAAS,GAAG,UAAU,GAAG,QAAQ,GAAG,WAAW,EAAE;AAClG,QAAM,0BAA0B;AAChC,QAAM,UAAU,KAAK,IAAI,GAAG,KAAK,IAAI,KAAK,KAAK,MAAM,KAAK,CAAC,CAAC;AAC5D,QAAM,SAAS,MAAM,MAAM,EAAE,QAAQ;AAAA,IACnC,KAAK;AAAA;AAAA,IAEL,MAAM,CAAC,OAAO;AAAA,EAChB,CAAC;AACD,QAAM,OAAO,OAAO;AACpB,MAAI,CAAC,KAAK,OAAQ,QAAO,EAAE,SAAS,GAAG,UAAU,GAAG,QAAQ,GAAG,WAAW,MAAM,2BAA2B,EAAE;AAE7G,MAAI,WAAW;AACf,MAAI,SAAS;AACb,QAAM,QAAQ,kBAAkB;AAChC,MAAI;AACF,UAAM,UAAU,MAAM,kBAAkB,KAAK,IAAI,SAAO,GAAG,IAAI,KAAK;AAAA;AAAA,EAAO,IAAI,IAAI,EAAE,CAAC;AACtF,eAAW,CAAC,OAAO,GAAG,KAAK,KAAK,QAAQ,GAAG;AACzC,YAAM,UAAU,IAAI,QAAQ,KAAK,EAAE,KAAK,GAAG,CAAC;AAC5C,YAAM,UAAU,EAAE;AAAA,QAChB;AAAA;AAAA;AAAA;AAAA,QAIA,CAAC,IAAI,IAAI,IAAI,WAAW,IAAI,eAAe,IAAI,OAAO,SAAS,KAAK;AAAA,MACtE;AACA,YAAM,MAAM,EAAE,QAAQ;AAAA,QACpB,KAAK;AAAA;AAAA,QAEL,MAAM,CAAC,wBAAwB,OAAO,IAAI,KAAI,oBAAI,KAAK,GAAE,YAAY,GAAG,IAAI,EAAE;AAAA,MAChF,CAAC;AACD,kBAAY;AAAA,IACd;AAAA,EACF,SAAS,OAAO;AACd,aAAS,KAAK,SAAS;AACvB,UAAM,WAAW,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,GAAG,MAAM,GAAG,GAAG;AACrF,eAAW,OAAO,KAAK,MAAM,QAAQ,GAAG;AACtC,YAAM,MAAM,EAAE,QAAQ;AAAA,QACpB,KAAK;AAAA,QACL,MAAM,CAAC,SAAS,IAAI,EAAE;AAAA,MACxB,CAAC,EAAE,MAAM,MAAM,MAAS;AAAA,IAC1B;AAAA,EACF;AACA,SAAO,EAAE,SAAS,KAAK,QAAQ,UAAU,QAAQ,WAAW,MAAM,2BAA2B,EAAE;AACjG;AAEA,eAAsB,6BAA8C;AAClE,QAAM,SAAS,MAAM,MAAM,EAAE,QAAQ,kGAAkG;AACvI,SAAO,OAAQ,OAAO,KAAK,CAAC,GAAgC,KAAK,CAAC;AACpE;AAEA,eAAsB,4BAA4B,OAAe,QAAQ,IAAkC;AACzG,QAAM,SAAS,oBAAI,IAAoB;AACvC,MAAI,CAAC,gCAAgC,KAAK,CAAC,MAAM,KAAK,EAAG,QAAO;AAChE,QAAM,0BAA0B;AAChC,QAAM,CAAC,MAAM,IAAI,MAAM,kBAAkB,CAAC,KAAK,CAAC;AAChD,MAAI,CAAC,OAAQ,QAAO;AACpB,QAAM,OAAO,MAAM,UAAU,EAAE;AAAA,IAC7B;AAAA;AAAA,IAEA,CAAC,IAAI,OAAO,KAAK,GAAG,CAAC,KAAK,KAAK,IAAI,GAAG,KAAK,IAAI,KAAK,KAAK,CAAC,CAAC;AAAA,EAC7D;AACA,aAAW,OAAO,KAAM,QAAO,IAAI,OAAO,IAAI,SAAS,GAAG,OAAO,IAAI,KAAK,CAAC;AAC3E,SAAO;AACT;","names":["document"]}
|
|
@@ -50,7 +50,7 @@ function renderInstallTerminal(options) {
|
|
|
50
50
|
"1/1 install surfaces ready",
|
|
51
51
|
colorize("Newest: any approved connection read can become an indexed Memory snapshot in one call. OAuth stays tenant-isolated and provider content is redacted and marked untrusted.", "lime", color),
|
|
52
52
|
"",
|
|
53
|
-
`${colorize("Tools", "cyan", color)} ${colorize("(
|
|
53
|
+
`${colorize("Tools", "cyan", color)} ${colorize("(259 MCP tools)", "muted", color)}`,
|
|
54
54
|
toolRow("search", ["harvest_paa", "search_serp", "maps_search", "maps_place_intel"], color),
|
|
55
55
|
toolRow("extract", ["extract_url", "map_site_urls", "extract_site", "audit_site", "directory_workflow"], color),
|
|
56
56
|
toolRow("build", ["create_editorial_reading_room", "rank_tracker_workflow", "portable HTML"], color),
|
|
@@ -106,4 +106,4 @@ function renderInstallTerminal(options) {
|
|
|
106
106
|
export {
|
|
107
107
|
renderInstallTerminal
|
|
108
108
|
};
|
|
109
|
-
//# sourceMappingURL=chunk-
|
|
109
|
+
//# sourceMappingURL=chunk-DCWXVAQT.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/install-terminal.ts"],"sourcesContent":["export interface InstallTerminalOptions {\n color?: boolean\n version: string\n apiKeyConfigured?: boolean\n}\n\ntype Tone = 'cyan' | 'lime' | 'amber' | 'red' | 'muted' | 'bold'\n\nconst CODES: Record<Tone | 'reset', string> = {\n reset: '\\x1b[0m',\n cyan: '\\x1b[36m',\n lime: '\\x1b[32m',\n amber: '\\x1b[33m',\n red: '\\x1b[31m',\n muted: '\\x1b[90m',\n bold: '\\x1b[1m',\n}\n\nfunction colorize(value: string, tone: Tone, enabled: boolean): string {\n if (!enabled) return value\n return `${CODES[tone]}${value}${CODES.reset}`\n}\n\nfunction toolRow(label: string, tools: string[], enabled: boolean): string {\n const padded = label.padEnd(9, ' ')\n return ` ${colorize(padded, 'muted', enabled)} ${tools.join(colorize(' . ', 'muted', enabled))}`\n}\n\nexport function renderInstallTerminal(options: InstallTerminalOptions): string {\n const color = options.color ?? true\n const apiKeyValue = options.apiKeyConfigured ? '$MCP_SCRAPER_API_KEY' : 'sk_live_your_key'\n const ascii = String.raw`\n __ __ ____ ____ \n| \\/ |/ ___| _ \\\n| |\\/| | | | |_) |\n| | | | |___| __/\n|_| |_|\\____|_|\n\n ____ ____ ____ _ ____ _____ ____\n/ ___| / ___| _ \\ / \\ | _ \\| ____| _ \\\n\\___ \\| | | |_) | / _ \\ | |_) | _| | |_) |\n ___) | |___| _ < / ___ \\| __/| |___| _ <\n|____/ \\____|_| \\_\\/_/ \\_\\_| |_____|_| \\_\\\n`\n\n const claudeCommand = [\n `MCP_SCRAPER_API_KEY=${apiKeyValue} npx -y -p mcp-scraper@latest \\\\`,\n ' mcp-scraper-cli agent install claude --apply',\n ].join('\\n')\n\n const codexConfig = [\n '[mcp_servers.mcp-scraper]',\n 'command = \"npx\"',\n 'args = [\"-y\", \"-p\", \"mcp-scraper@latest\", \"mcp-scraper\"]',\n `env = { MCP_SCRAPER_API_KEY = \"${apiKeyValue}\" }`,\n ].join('\\n')\n\n return [\n colorize(`mcp-scraper v${options.version}`, 'bold', color),\n colorize('> mcp-scraper-install', 'muted', color),\n colorize(ascii, 'amber', color),\n `${colorize('MCP Scraper Agent', 'cyan', color)} . v${options.version} . mcpscraper.dev`,\n '1/1 install surfaces ready',\n colorize('Newest: any approved connection read can become an indexed Memory snapshot in one call. OAuth stays tenant-isolated and provider content is redacted and marked untrusted.', 'lime', color),\n '',\n `${colorize('Tools', 'cyan', color)} ${colorize('(
|
|
1
|
+
{"version":3,"sources":["../src/install-terminal.ts"],"sourcesContent":["export interface InstallTerminalOptions {\n color?: boolean\n version: string\n apiKeyConfigured?: boolean\n}\n\ntype Tone = 'cyan' | 'lime' | 'amber' | 'red' | 'muted' | 'bold'\n\nconst CODES: Record<Tone | 'reset', string> = {\n reset: '\\x1b[0m',\n cyan: '\\x1b[36m',\n lime: '\\x1b[32m',\n amber: '\\x1b[33m',\n red: '\\x1b[31m',\n muted: '\\x1b[90m',\n bold: '\\x1b[1m',\n}\n\nfunction colorize(value: string, tone: Tone, enabled: boolean): string {\n if (!enabled) return value\n return `${CODES[tone]}${value}${CODES.reset}`\n}\n\nfunction toolRow(label: string, tools: string[], enabled: boolean): string {\n const padded = label.padEnd(9, ' ')\n return ` ${colorize(padded, 'muted', enabled)} ${tools.join(colorize(' . ', 'muted', enabled))}`\n}\n\nexport function renderInstallTerminal(options: InstallTerminalOptions): string {\n const color = options.color ?? true\n const apiKeyValue = options.apiKeyConfigured ? '$MCP_SCRAPER_API_KEY' : 'sk_live_your_key'\n const ascii = String.raw`\n __ __ ____ ____ \n| \\/ |/ ___| _ \\\n| |\\/| | | | |_) |\n| | | | |___| __/\n|_| |_|\\____|_|\n\n ____ ____ ____ _ ____ _____ ____\n/ ___| / ___| _ \\ / \\ | _ \\| ____| _ \\\n\\___ \\| | | |_) | / _ \\ | |_) | _| | |_) |\n ___) | |___| _ < / ___ \\| __/| |___| _ <\n|____/ \\____|_| \\_\\/_/ \\_\\_| |_____|_| \\_\\\n`\n\n const claudeCommand = [\n `MCP_SCRAPER_API_KEY=${apiKeyValue} npx -y -p mcp-scraper@latest \\\\`,\n ' mcp-scraper-cli agent install claude --apply',\n ].join('\\n')\n\n const codexConfig = [\n '[mcp_servers.mcp-scraper]',\n 'command = \"npx\"',\n 'args = [\"-y\", \"-p\", \"mcp-scraper@latest\", \"mcp-scraper\"]',\n `env = { MCP_SCRAPER_API_KEY = \"${apiKeyValue}\" }`,\n ].join('\\n')\n\n return [\n colorize(`mcp-scraper v${options.version}`, 'bold', color),\n colorize('> mcp-scraper-install', 'muted', color),\n colorize(ascii, 'amber', color),\n `${colorize('MCP Scraper Agent', 'cyan', color)} . v${options.version} . mcpscraper.dev`,\n '1/1 install surfaces ready',\n colorize('Newest: any approved connection read can become an indexed Memory snapshot in one call. OAuth stays tenant-isolated and provider content is redacted and marked untrusted.', 'lime', color),\n '',\n `${colorize('Tools', 'cyan', color)} ${colorize('(259 MCP tools)', 'muted', color)}`,\n toolRow('search', ['harvest_paa', 'search_serp', 'maps_search', 'maps_place_intel'], color),\n toolRow('extract', ['extract_url', 'map_site_urls', 'extract_site', 'audit_site', 'directory_workflow'], color),\n toolRow('build', ['create_editorial_reading_room', 'rank_tracker_workflow', 'portable HTML'], color),\n toolRow('media', ['youtube_harvest', 'youtube_transcribe', 'facebook_ad_search', 'facebook_page_intel', 'facebook_ad_transcribe', 'facebook_video_transcribe', 'instagram_profile_content', 'instagram_media_download', 'reddit_thread'], color),\n toolRow('browser', ['serp_identity_create', 'serp_identity_list', 'browser_open', 'browser_profile_connect', 'browser_profile_list', 'browser_close', 'browser_screenshot', 'browser_read', 'browser_locate', 'browser_replay_mark', 'browser_replay_annotate'], color),\n toolRow('connect', ['list_service_connections', 'describe_service_connection_tool', 'import_service_connection_to_memory', 'export_connected_service_data', 'renew_connected_data_download', 'read_service_connection', 'call_service_connection_action'], color),\n toolRow('commons', ['commons_search_entities', 'commons_get_entity_linkset', 'commons_prepare_entity', 'commons_submit_entity', 'commons_prepare_publication', 'commons_claim_publication', 'commons_publish_editorial', 'commons_get_publication'], color),\n toolRow('account', ['credits_info', 'reports', 'MCP resources'], color),\n toolRow('memory', ['memory-put', 'memory-get', 'memory-search', 'list-vaults', 'record-fact', 'list-scheduled-actions'], color),\n `${colorize('Workflows', 'cyan', color)} ${colorize('(MCP + CLI + API)', 'muted', color)}`,\n toolRow('route', ['workflow_list', 'workflow_suggest', 'workflow_run', 'workflow_step', 'workflow_status', 'workflow_artifact_read'], color),\n toolRow('seo', ['directory', 'agent-packet', 'competitive audit', 'map/serp comparison', 'PAA/AIO briefs', 'scheduled runs'], color),\n '',\n colorize('Usage tips:', 'amber', color),\n 'Run mcp-scraper-install for this visible card. Run mcp-scraper-cli for setup utilities and subcommands.',\n 'Run mcp-scraper in a human terminal to print this card; MCP clients get the same command as a silent stdio server.',\n 'Explicit card command: npx -y -p mcp-scraper@latest mcp-scraper-install',\n 'Hosted browser sessions use direct/no-proxy egress by default.',\n 'Customer auth setup: run browser_profile_connect, send the watch_url, let the user sign in, then call browser_profile_list until AUTHENTICATED.',\n 'Connected account ranges: call export_connected_service_data once. It handles Gmail, Calendar, Zoom, and Resend pagination; do not loop read_service_connection over individual records.',\n 'Connected account RAG: call import_service_connection_to_memory for one bounded approved read. It writes a redacted, untrusted snapshot to a stable Memory path and embeds it for search.',\n 'Stack logins / reconnect: run browser_profile_connect again with the same profile name and another domain to add accounts or refresh a login.',\n 'Start with workflow_suggest for broad jobs like market analysis, ICP research, CRO audits, brand briefs, content gaps, and AI visibility.',\n 'For MCP clients, use mcp-scraper so one install can mix SERP, Maps, browser, reports, and saved MCP resources.',\n 'If you hit the concurrency limit, add 2 browsers for $5/month with mcp-scraper-cli billing concurrency checkout.',\n '',\n `${colorize('Ready.', 'lime', color)} Install the combined MCP server with one command:`,\n '',\n colorize('Setup doctor', 'amber', color),\n 'npx -y -p mcp-scraper@latest mcp-scraper-cli doctor',\n '',\n colorize('Hosted profile setup', 'amber', color),\n 'In your MCP client, call browser_profile_connect with email=\"seo@example.com\" and domain=\"chatgpt.com\".',\n 'Give the returned watch_url to the user. After they sign in, call browser_profile_list, then browser_open with the returned profile. Add more logins by calling browser_profile_connect again with the same profile and a new domain.',\n '',\n colorize('Claude Code one-command setup', 'amber', color),\n claudeCommand,\n 'Then fully exit Claude Code and open a new Claude terminal. Check with: claude mcp list',\n '',\n colorize('Codex config', 'amber', color),\n codexConfig,\n '',\n colorize('Claude Desktop Extension', 'amber', color),\n 'Download: https://mcpscraper.dev/downloads/mcp-scraper.mcpb',\n '',\n colorize('Safety note:', 'muted', color),\n 'mcp-scraper prints this card only when stdin/stdout are an interactive TTY. In MCP clients it writes only JSON-RPC to stdout.',\n 'Use --stdio or MCP_SCRAPER_FORCE_STDIO=1 to force server mode from a terminal.',\n '',\n ].join('\\n')\n}\n"],"mappings":";AAQA,IAAM,QAAwC;AAAA,EAC5C,OAAO;AAAA,EACP,MAAM;AAAA,EACN,MAAM;AAAA,EACN,OAAO;AAAA,EACP,KAAK;AAAA,EACL,OAAO;AAAA,EACP,MAAM;AACR;AAEA,SAAS,SAAS,OAAe,MAAY,SAA0B;AACrE,MAAI,CAAC,QAAS,QAAO;AACrB,SAAO,GAAG,MAAM,IAAI,CAAC,GAAG,KAAK,GAAG,MAAM,KAAK;AAC7C;AAEA,SAAS,QAAQ,OAAe,OAAiB,SAA0B;AACzE,QAAM,SAAS,MAAM,OAAO,GAAG,GAAG;AAClC,SAAO,KAAK,SAAS,QAAQ,SAAS,OAAO,CAAC,IAAI,MAAM,KAAK,SAAS,SAAS,SAAS,OAAO,CAAC,CAAC;AACnG;AAEO,SAAS,sBAAsB,SAAyC;AAC7E,QAAM,QAAQ,QAAQ,SAAS;AAC/B,QAAM,cAAc,QAAQ,mBAAmB,yBAAyB;AACxE,QAAM,QAAQ,OAAO;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAcrB,QAAM,gBAAgB;AAAA,IACpB,uBAAuB,WAAW;AAAA,IAClC;AAAA,EACF,EAAE,KAAK,IAAI;AAEX,QAAM,cAAc;AAAA,IAClB;AAAA,IACA;AAAA,IACA;AAAA,IACA,kCAAkC,WAAW;AAAA,EAC/C,EAAE,KAAK,IAAI;AAEX,SAAO;AAAA,IACL,SAAS,gBAAgB,QAAQ,OAAO,IAAI,QAAQ,KAAK;AAAA,IACzD,SAAS,yBAAyB,SAAS,KAAK;AAAA,IAChD,SAAS,OAAO,SAAS,KAAK;AAAA,IAC9B,GAAG,SAAS,qBAAqB,QAAQ,KAAK,CAAC,SAAS,QAAQ,OAAO;AAAA,IACvE;AAAA,IACA,SAAS,8KAA8K,QAAQ,KAAK;AAAA,IACpM;AAAA,IACA,GAAG,SAAS,SAAS,QAAQ,KAAK,CAAC,KAAK,SAAS,mBAAmB,SAAS,KAAK,CAAC;AAAA,IACnF,QAAQ,UAAU,CAAC,eAAe,eAAe,eAAe,kBAAkB,GAAG,KAAK;AAAA,IAC1F,QAAQ,WAAW,CAAC,eAAe,iBAAiB,gBAAgB,cAAc,oBAAoB,GAAG,KAAK;AAAA,IAC9G,QAAQ,SAAS,CAAC,iCAAiC,yBAAyB,eAAe,GAAG,KAAK;AAAA,IACnG,QAAQ,SAAS,CAAC,mBAAmB,sBAAsB,sBAAsB,uBAAuB,0BAA0B,6BAA6B,6BAA6B,4BAA4B,eAAe,GAAG,KAAK;AAAA,IAC/O,QAAQ,WAAW,CAAC,wBAAwB,sBAAsB,gBAAgB,2BAA2B,wBAAwB,iBAAiB,sBAAsB,gBAAgB,kBAAkB,uBAAuB,yBAAyB,GAAG,KAAK;AAAA,IACtQ,QAAQ,WAAW,CAAC,4BAA4B,oCAAoC,uCAAuC,iCAAiC,iCAAiC,2BAA2B,gCAAgC,GAAG,KAAK;AAAA,IAChQ,QAAQ,WAAW,CAAC,2BAA2B,8BAA8B,0BAA0B,yBAAyB,+BAA+B,6BAA6B,6BAA6B,yBAAyB,GAAG,KAAK;AAAA,IAC1P,QAAQ,WAAW,CAAC,gBAAgB,WAAW,eAAe,GAAG,KAAK;AAAA,IACtE,QAAQ,UAAU,CAAC,cAAc,cAAc,iBAAiB,eAAe,eAAe,wBAAwB,GAAG,KAAK;AAAA,IAC9H,GAAG,SAAS,aAAa,QAAQ,KAAK,CAAC,KAAK,SAAS,qBAAqB,SAAS,KAAK,CAAC;AAAA,IACzF,QAAQ,SAAS,CAAC,iBAAiB,oBAAoB,gBAAgB,iBAAiB,mBAAmB,wBAAwB,GAAG,KAAK;AAAA,IAC3I,QAAQ,OAAO,CAAC,aAAa,gBAAgB,qBAAqB,uBAAuB,kBAAkB,gBAAgB,GAAG,KAAK;AAAA,IACnI;AAAA,IACA,SAAS,eAAe,SAAS,KAAK;AAAA,IACtC;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,GAAG,SAAS,UAAU,QAAQ,KAAK,CAAC;AAAA,IACpC;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA,SAAS,wBAAwB,SAAS,KAAK;AAAA,IAC/C;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,iCAAiC,SAAS,KAAK;AAAA,IACxD;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA,SAAS,4BAA4B,SAAS,KAAK;AAAA,IACnD;AAAA,IACA;AAAA,IACA,SAAS,gBAAgB,SAAS,KAAK;AAAA,IACvC;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI;AACb;","names":[]}
|
|
@@ -17,7 +17,7 @@ import {
|
|
|
17
17
|
} from "./chunk-4LINSJNI.js";
|
|
18
18
|
import {
|
|
19
19
|
PACKAGE_VERSION
|
|
20
|
-
} from "./chunk-
|
|
20
|
+
} from "./chunk-5BODYBIP.js";
|
|
21
21
|
import {
|
|
22
22
|
createPrivateArtifact,
|
|
23
23
|
privateArtifactOwnerId,
|
|
@@ -30,7 +30,7 @@ import {
|
|
|
30
30
|
PAA_QUESTION_CREDITS,
|
|
31
31
|
PAGE_SCRAPE_CREDITS,
|
|
32
32
|
SERP_SEARCH_CREDITS
|
|
33
|
-
} from "./chunk-
|
|
33
|
+
} from "./chunk-2XUXVKT4.js";
|
|
34
34
|
import {
|
|
35
35
|
PUBLIC_ERROR_CODES,
|
|
36
36
|
buildPublicErrorEnvelope,
|
|
@@ -342,6 +342,8 @@ seam is noted so you can chain them.
|
|
|
342
342
|
- Whole site -> **extract_site** (takes a url). It durably retains complete per-page JSON, acquired HTML,
|
|
343
343
|
and Markdown. Poll **check_site_export**, then call **site_export_read** for the manifest or a page view;
|
|
344
344
|
call **site_export_image** for downloaded image IDs.
|
|
345
|
+
- JavaScript-rendered page overlap/cannibalization -> **analyze_site_similarity**. It returns dedicated
|
|
346
|
+
raw-cosine page pairs, corpus percentiles, threshold clusters, and rendered-page artifacts.
|
|
345
347
|
- Wayback replay URLs work with the same tools: \`extract_url\` removes playback chrome and can return
|
|
346
348
|
a featured image; \`extract_site\` batches nearby archived HTML captures for the replayed site.
|
|
347
349
|
- For multiple archive months, pass \`extract_site.wayback\` with explicit \`months\` or a \`from\`/\`to\`
|
|
@@ -2589,7 +2591,7 @@ async function formatExtractSite(raw, input, ctx) {
|
|
|
2589
2591
|
const parsed = parseData(raw);
|
|
2590
2592
|
if ("error" in parsed) return formattedErrorResult(parsed.error);
|
|
2591
2593
|
const started = formatBackgroundJobStarted(
|
|
2592
|
-
"Multi-Page Site Content Crawl",
|
|
2594
|
+
input.toolLabel ?? "Multi-Page Site Content Crawl",
|
|
2593
2595
|
parsed.data,
|
|
2594
2596
|
{
|
|
2595
2597
|
requested: input.delivery ?? "auto",
|
|
@@ -4140,7 +4142,8 @@ function formatLeadListImport(raw) {
|
|
|
4140
4142
|
const map = Object.keys(detailedMap).length > 0 ? detailedMap : Object.fromEntries(Object.entries(simpleMap).map(([field, header]) => [field, {
|
|
4141
4143
|
header: header == null ? null : String(header),
|
|
4142
4144
|
confidence: header == null ? 0 : 1,
|
|
4143
|
-
reason: "server_suggestion"
|
|
4145
|
+
reason: "server_suggestion",
|
|
4146
|
+
accepted: header != null
|
|
4144
4147
|
}]));
|
|
4145
4148
|
const mappingLines = Object.entries(map).slice(0, 12).map(([field, suggestion]) => {
|
|
4146
4149
|
const record = structuredRecord(suggestion);
|
|
@@ -4995,6 +4998,7 @@ ${rows}` : ""
|
|
|
4995
4998
|
var ESSENTIAL_OUTPUT_SCHEMA_TOOLS = /* @__PURE__ */ new Set([
|
|
4996
4999
|
"extract_url",
|
|
4997
5000
|
"extract_site",
|
|
5001
|
+
"analyze_site_similarity",
|
|
4998
5002
|
"audit_site",
|
|
4999
5003
|
"check_site_export",
|
|
5000
5004
|
"site_export_read",
|
|
@@ -6311,6 +6315,13 @@ var ExtractSiteInputSchema = {
|
|
|
6311
6315
|
preserveMedia: z5.boolean().default(false).describe("Include supported images in the export bundle. This is the preferred replacement for downloadImages."),
|
|
6312
6316
|
downloadImages: z5.boolean().optional().describe("Deprecated alias for preserveMedia. Omit when using preserveMedia; when omitted, image preservation defaults to false.")
|
|
6313
6317
|
};
|
|
6318
|
+
var AnalyzeSiteSimilarityInputSchema = {
|
|
6319
|
+
url: WebsiteUrlOrDomainSchema.describe("Public live site to render and compare. Bare domains default to https://."),
|
|
6320
|
+
maxPages: z5.number().int().min(2).max(500).default(100).describe("Maximum rendered pages to compare; default 100, maximum 500."),
|
|
6321
|
+
similarityThreshold: z5.number().min(0).max(1).default(0.9).describe("Minimum raw cosine score retained in the pair table; default 0.90."),
|
|
6322
|
+
similarityMaxPairs: z5.number().int().min(1).max(5e4).default(1e4).describe("Maximum scored pairs retained, highest first; default 10,000."),
|
|
6323
|
+
idempotencyKey: z5.string().trim().min(8).max(200).describe("Required unique opaque ID for this intended analysis. Reuse only when retrying the same call; use a new value for an intentional rerun.")
|
|
6324
|
+
};
|
|
6314
6325
|
var AuditSiteInputSchema = {
|
|
6315
6326
|
url: WebsiteUrlOrDomainSchema.describe("Public website URL or domain for a full technical SEO audit (issues, link graph, indexability, headings, images). Bare domains default to https://. For plain content use extract_site instead."),
|
|
6316
6327
|
maxPages: z5.number().int().min(1).max(1e4).optional().describe("Maximum pages to crawl and audit. MCP audits always run as durable background exports and return a jobId; poll check_site_export for the hosted audit ZIP."),
|
|
@@ -6323,17 +6334,17 @@ var AuditSiteInputSchema = {
|
|
|
6323
6334
|
downloadImages: z5.boolean().optional().describe("Deprecated alias for preserveMedia. Omit when using preserveMedia; when omitted, image preservation defaults to false.")
|
|
6324
6335
|
};
|
|
6325
6336
|
var CheckSiteExportInputSchema = {
|
|
6326
|
-
jobId: z5.string().min(1).describe("The jobId returned by extract_site or audit_site. Poll until status is complete, partial, or failed; partial jobs still return
|
|
6337
|
+
jobId: z5.string().min(1).describe("The jobId returned by extract_site, analyze_site_similarity, or audit_site. Poll until status is complete, partial, or failed; partial jobs still return successful content and failure details.")
|
|
6327
6338
|
};
|
|
6328
6339
|
var SiteExportReadInputSchema = {
|
|
6329
|
-
jobId: z5.string().min(1).describe("Site export job ID returned by extract_site or audit_site."),
|
|
6340
|
+
jobId: z5.string().min(1).describe("Site export job ID returned by extract_site, analyze_site_similarity, or audit_site."),
|
|
6330
6341
|
pageId: z5.string().regex(/^[a-f0-9]{64}$/).optional().describe("Page ID returned by a manifest read. Omit to list the export manifest."),
|
|
6331
6342
|
format: z5.enum(["manifest", "json", "html", "markdown"]).default("manifest").describe("manifest lists page/image IDs; JSON, HTML, and Markdown read one page representation."),
|
|
6332
6343
|
offset: z5.number().int().min(0).default(0).describe("UTF-8 byte offset. Continue from nextOffset until it is null."),
|
|
6333
6344
|
maxBytes: z5.number().int().min(1).max(1e6).default(64e3).describe("Maximum UTF-8 bytes returned in this window.")
|
|
6334
6345
|
};
|
|
6335
6346
|
var SiteExportImageInputSchema = {
|
|
6336
|
-
jobId: z5.string().min(1).describe("Site export job ID returned by extract_site or audit_site."),
|
|
6347
|
+
jobId: z5.string().min(1).describe("Site export job ID returned by extract_site, analyze_site_similarity, or audit_site."),
|
|
6337
6348
|
imageId: z5.string().regex(/^[a-f0-9]{64}$/).describe("Downloaded image ID returned by a site_export_read manifest.")
|
|
6338
6349
|
};
|
|
6339
6350
|
var ArchiveReadInputSchema = {
|
|
@@ -7161,7 +7172,8 @@ var DirectoryWorkflowOutputSchema = {
|
|
|
7161
7172
|
var LeadSuggestedColumnOutputSchema = z5.object({
|
|
7162
7173
|
header: NullableString,
|
|
7163
7174
|
confidence: z5.number().min(0).max(1),
|
|
7164
|
-
reason: z5.string()
|
|
7175
|
+
reason: z5.string(),
|
|
7176
|
+
accepted: z5.boolean()
|
|
7165
7177
|
}).strict();
|
|
7166
7178
|
var LeadColumnMapSuggestionOutputSchema = z5.object({
|
|
7167
7179
|
name: LeadSuggestedColumnOutputSchema,
|
|
@@ -10207,6 +10219,21 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
10207
10219
|
await formatExtractSite(await executor.extractSite(input), input, ctx),
|
|
10208
10220
|
requestContext
|
|
10209
10221
|
));
|
|
10222
|
+
server.registerTool("analyze_site_similarity", {
|
|
10223
|
+
title: "Rendered Site Content Similarity",
|
|
10224
|
+
description: "Find overlapping or competing live pages. Forces JavaScript rendering, retains bounded inert DOM plus clean Markdown, and returns raw-cosine pairs, corpus percentiles, clusters, and table-ready files in a private ZIP. Poll check_site_export; the scores measure content overlap, not intent or proof of cannibalization.",
|
|
10225
|
+
inputSchema: AnalyzeSiteSimilarityInputSchema,
|
|
10226
|
+
outputSchema: recordOutputSchema("analyze_site_similarity", ExtractSiteOutputSchema),
|
|
10227
|
+
annotations: { ...liveWebToolAnnotations("Rendered Site Content Similarity"), readOnlyHint: false }
|
|
10228
|
+
}, async (input, requestContext) => tasks.taskify(
|
|
10229
|
+
"site_export",
|
|
10230
|
+
await formatExtractSite(
|
|
10231
|
+
await executor.analyzeSiteSimilarity(input),
|
|
10232
|
+
{ ...input, toolLabel: "Rendered Site Content Similarity" },
|
|
10233
|
+
ctx
|
|
10234
|
+
),
|
|
10235
|
+
requestContext
|
|
10236
|
+
));
|
|
10210
10237
|
server.registerTool("audit_site", {
|
|
10211
10238
|
title: "Technical SEO Audit",
|
|
10212
10239
|
description: `Run a full technical SEO audit (Screaming-Frog-style) on a public website: on-page issues, internal link graph, indexability, heading/image analysis. Pass a new idempotencyKey for each intended audit and reuse it only when retrying that call. Every MCP audit starts a durable export; poll check_site_export for discovered, attempted, successful, failed, and remaining counts plus ${fileBehavior("the saved ZIP.", "the owner-scoped downloadable ZIP.")} Use extract_site instead for plain page content.`,
|
|
@@ -10220,7 +10247,7 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
10220
10247
|
));
|
|
10221
10248
|
server.registerTool("check_site_export", {
|
|
10222
10249
|
title: "Check Site Export",
|
|
10223
|
-
description: "Poll a background extract_site or audit_site job. Reports
|
|
10250
|
+
description: "Poll a background extract_site, analyze_site_similarity, or audit_site job. Reports page counters and a stable public error envelope when terminal. Complete and partial jobs expose direct AI readback and a downloadable ZIP.",
|
|
10224
10251
|
inputSchema: CheckSiteExportInputSchema,
|
|
10225
10252
|
outputSchema: recordOutputSchema("check_site_export", CheckSiteExportOutputSchema),
|
|
10226
10253
|
annotations: { ...liveWebToolAnnotations("Check Site Export"), readOnlyHint: false }
|
|
@@ -11219,6 +11246,19 @@ var HttpMcpToolExecutor = class {
|
|
|
11219
11246
|
const { idempotencyKey, ...body } = input;
|
|
11220
11247
|
return this.callSiteExtractStart("extract_site", { ...body, background: true }, idempotencyKey);
|
|
11221
11248
|
}
|
|
11249
|
+
analyzeSiteSimilarity(input) {
|
|
11250
|
+
const { idempotencyKey, similarityThreshold, similarityMaxPairs, ...body } = input;
|
|
11251
|
+
return this.callSiteExtractStart("analyze_site_similarity", {
|
|
11252
|
+
...body,
|
|
11253
|
+
background: true,
|
|
11254
|
+
formats: ["markdown", "links", "json"],
|
|
11255
|
+
renderJavaScript: true,
|
|
11256
|
+
captureRenderedDom: true,
|
|
11257
|
+
semanticSimilarity: true,
|
|
11258
|
+
similarityThreshold,
|
|
11259
|
+
similarityMaxPairs
|
|
11260
|
+
}, idempotencyKey);
|
|
11261
|
+
}
|
|
11222
11262
|
auditSite(input) {
|
|
11223
11263
|
const { idempotencyKey, ...body } = input;
|
|
11224
11264
|
const requestBody = {
|
|
@@ -16670,4 +16710,4 @@ export {
|
|
|
16670
16710
|
ScheduledResultsMcpExecutor,
|
|
16671
16711
|
registerScheduledResultsMcpTools
|
|
16672
16712
|
};
|
|
16673
|
-
//# sourceMappingURL=chunk-
|
|
16713
|
+
//# sourceMappingURL=chunk-PSKRQDGN.js.map
|