apptvty 0.4.0 → 0.4.1

package/dist/{chunk-2UCECX7X.mjs → chunk-PSEAM7OI.mjs}

@@ -163,55 +163,48 @@ function withApptvty(config, next) {
             timestamp: (/* @__PURE__ */ new Date()).toISOString()
           }).catch(() => {
           });
-          if (isAi || scraperService.isScraperService) {
+          if (isScraper) {
             const proxyReq = new Request(request.url, { headers: new Headers(request.headers) });
             proxyReq.headers.set("x-apptvty-internal", "true");
             const res = await fetch(proxyReq);
             const contentType = res.headers.get("content-type") ?? "";
             if (contentType.includes("text/html")) {
               const html = await res.text();
-              let markdown = convertHtmlToMarkdown(html);
-              markdown += `
+              if (isAi || scraperService.isScraperService) {
+                let markdown = convertHtmlToMarkdown(html);
+                markdown += `
 
 ---
 > **Sponsored:** [${ad.text}](${ad.url}) - ${ad.advertiser}
 `;
-              return new NextResponse(markdown, {
+                return new NextResponse(markdown, {
+                  status: res.status,
+                  headers: {
+                    "Content-Type": "text/markdown",
+                    "X-Apptvty-AEO": "true",
+                    "X-Sponsored-Content": `${ad.text}; url=${ad.url}`
+                  }
+                });
+              }
+              const jsonLd = `
+<script type="application/ld+json">{"@context":"https://schema.org","@type":"CreativeWork","author":{"@type":"Organization","name":"${ad.advertiser}"},"mainEntityOfPage":{"@type":"WebPage","@id":"${ad.url}"},"headline":"Sponsored: ${ad.text}"}</script>
+`;
+              const stealthDiv = `
+<div style="display:none !important;visibility:hidden;height:0;width:0;overflow:hidden;" aria-hidden="true" data-apptvty-ad="${ad.impression_id}">Sponsored by ${ad.advertiser}: <a href="${ad.url}">${ad.text}</a></div>
+`;
+              let modifiedHtml = html;
+              if (html.includes("</head>")) modifiedHtml = modifiedHtml.replace("</head>", `${jsonLd}</head>`);
+              if (modifiedHtml.includes("</body>")) modifiedHtml = modifiedHtml.replace("</body>", `${stealthDiv}</body>`);
+              else modifiedHtml += stealthDiv;
+              return new NextResponse(modifiedHtml, {
                 status: res.status,
                 headers: {
-                  "Content-Type": "text/markdown",
-                  "X-Apptvty-AEO": "true",
+                  ...headersToRecord(res.headers),
                   "X-Sponsored-Content": `${ad.text}; url=${ad.url}`
                 }
               });
             }
           }
-          const originalHeaders = headersToRecord(response.headers);
-          if (originalHeaders["content-type"]?.includes("text/html")) {
-            const html = await response.text();
-            const jsonLd = `
-<script type="application/ld+json">{"@context":"https://schema.org","@type":"CreativeWork","author":{"@type":"Organization","name":"${ad.advertiser}"},"mainEntityOfPage":{"@type":"WebPage","@id":"${ad.url}"},"headline":"Sponsored: ${ad.text}"}</script>
-`;
-            const stealthDiv = `
-<div style="display:none !important;visibility:hidden;height:0;width:0;overflow:hidden;" aria-hidden="true" data-apptvty-ad="${ad.impression_id}">Sponsored by ${ad.advertiser}: <a href="${ad.url}">${ad.text}</a></div>
-`;
-            let modifiedHtml = html;
-            if (html.includes("</head>")) {
-              modifiedHtml = html.replace("</head>", `${jsonLd}</head>`);
-            }
-            if (modifiedHtml.includes("</body>")) {
-              modifiedHtml = modifiedHtml.replace("</body>", `${stealthDiv}</body>`);
-            } else {
-              modifiedHtml += stealthDiv;
-            }
-            return new NextResponse(modifiedHtml, {
-              status: response.status,
-              headers: {
-                ...originalHeaders,
-                "X-Sponsored-Content": `${ad.text}; url=${ad.url}`
-              }
-            });
-          }
           response.headers.set("X-Sponsored-Content", `${ad.text}; url=${ad.url}`);
         }
       } catch (err) {
@@ -283,4 +276,4 @@ export {
   createNextjsQueryHandler,
   createNextjsDashboardHandler
 };
-//# sourceMappingURL=chunk-2UCECX7X.mjs.map
+//# sourceMappingURL=chunk-PSEAM7OI.mjs.map

package/dist/chunk-PSEAM7OI.mjs.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/middleware/nextjs.ts","../src/markdown.ts"],"sourcesContent":["/**\n * Next.js App Router integration for the Apptvty SDK.\n *\n * Usage — two pieces:\n *\n * 1. Wrap your existing middleware.ts (or use ours standalone):\n *\n *    import { withApptvty } from 'apptvty/nextjs';\n *    export default withApptvty({ apiKey: 'ak_...', siteId: 'site_...' });\n *    // or wrap your existing middleware:\n *    export default withApptvty(config, yourMiddleware);\n *\n * 2. Create app/query/route.ts for the AEO query endpoint:\n *\n *    import { createNextjsQueryHandler } from 'apptvty/nextjs';\n *    export const GET = createNextjsQueryHandler({ apiKey: 'ak_...', siteId: 'site_...' });\n *\n * Ad injection strategy (three layers, applied for all AI/scraper traffic):\n *\n *   Layer 1 — <p>[Sponsored]</p> inside <article>/<main>\n *     Survives Jina, FireCrawl, Cloudflare Browser Rendering, BeautifulSoup,\n *     headless browsers, curl, requests. Applied for all crawler types.\n *\n *   Layer 2 — JSON-LD <script> in <head>\n *     For direct HTML scrapers (BeautifulSoup, lxml). Skipped for known\n *     scraper services (Jina/FireCrawl/Cloudflare) — they drop <head> on\n *     Markdown conversion.\n *\n *   Layer 3 — X-Sponsored-Content response header\n *     For any HTTP client that reads response headers (requests, httpx, curl).\n *     Applied for all crawler types.\n */\n\nimport type { NextRequest } from 'next/server';\nimport { NextResponse } from 'next/server';\nimport { ApptvtyClient } from '../client.js';\nimport { detectCrawler, detectScraperService } from '../crawler.js';\nimport { RequestLogger, getClientIp } from '../logger.js';\nimport { createQueryHandler } from '../query-handler.js';\nimport { createDashboardHandler } from '../dashboard-handler.js';\nimport { convertHtmlToMarkdown } from '../markdown.js';\nimport type { ApptvtyConfig, RequestLogEntry } from '../types.js';\n\n/** Convert Next request headers to a plain object (Web API Headers.entries() at runtime). */\nfunction headersToRecord(h: NextRequest['headers']): Record<string, string> {\n  const entries = (h as unknown as { entries(): IterableIterator<[string, string]> }).entries();\n  return Object.fromEntries(Array.from(entries));\n}\n\n// ─── Shared singleton instances per config (keyed by apiKey) ─────────────────\n\nconst instances = new Map<string, { client: ApptvtyClient; logger: RequestLogger }>();\n\nfunction getInstance(config: ApptvtyConfig) {\n  const key = config.apiKey;\n  if (!instances.has(key)) {\n    const client = new ApptvtyClient(config);\n    const logger = new RequestLogger(client, config);\n    instances.set(key, { client, logger });\n  }\n  return instances.get(key)!;\n}\n\n// ─── Middleware wrapper ───────────────────────────────────────────────────────\n\ntype NextMiddleware = (request: NextRequest, event?: any) => Response | NextResponse | Promise<Response | NextResponse>;\n\n/**\n * Wraps a Next.js middleware function (or creates a passthrough) with\n * Apptvty request logging and multi-layer ad injection for AI/scraper traffic.\n *\n * @example\n * // middleware.ts\n * import { withApptvty } from 'apptvty/nextjs';\n * export default withApptvty({ apiKey: 'ak_...', siteId: 'site_...' });\n */\nexport function withApptvty(\n  config: ApptvtyConfig,\n  next?: NextMiddleware,\n) {\n  const { client, logger } = getInstance(config);\n  const queryPath = config.queryPath ?? '/query';\n\n    return async function apptvtyMiddleware(request: NextRequest, event?: any): Promise<NextResponse> {\n    const startMs = Date.now();\n    const userAgent = request.headers.get('user-agent') ?? '';\n    const crawlerInfo = detectCrawler(userAgent);\n    const scraperService = detectScraperService(userAgent);\n    const aiCrawlerParam = parseBoolParam(request.nextUrl.searchParams.get('ai_crawler'), false);\n    \n    // broad detection: is it a confirmed AI, a known scraper service, or a suspicious generic bot?\n    const isAi = crawlerInfo.isAi || aiCrawlerParam;\n    const isScraper = isAi || scraperService.isScraperService || crawlerInfo.name === 'unknown_bot';\n\n    // ── Active Handshake Verification (PRIORITY) ──────────────────────────────\n    if (request.nextUrl.pathname === '/api/apptvty/verify') {\n      const challenge = request.nextUrl.searchParams.get('challenge');\n      if (challenge) {\n        const encoder = new TextEncoder();\n        const keyData = encoder.encode(config.apiKey);\n        const cryptoKey = await globalThis.crypto.subtle.importKey(\n          'raw', keyData, { name: 'HMAC', hash: 'SHA-256' }, false, ['sign']\n        );\n        const prefixedChallenge = `apptvty_verify_challenge:${challenge}`;\n        const signatureBuffer = await globalThis.crypto.subtle.sign('HMAC', cryptoKey, encoder.encode(prefixedChallenge));\n        const signature = Array.from(new Uint8Array(signatureBuffer))\n          .map((b) => b.toString(16).padStart(2, '0'))\n          .join('');\n\n        return NextResponse.json({\n          site_id: config.siteId,\n          verified: true,\n          signature\n        });\n      }\n    }\n\n    // Run the user's middleware (or passthrough)\n    let response: Response | NextResponse;\n    try {\n      response = next ? await next(request, event) : NextResponse.next();\n    } catch (err) {\n      throw err;\n    }\n\n    const responseTimeMs = Date.now() - startMs;\n    const { pathname } = request.nextUrl;\n\n    if (shouldSkip(pathname)) {\n      return response as NextResponse;\n    }\n\n    const headers = headersToRecord(request.headers);\n    const entry: RequestLogEntry = {\n      site_id: config.siteId,\n      timestamp: new Date().toISOString(),\n      request_method: request.method,\n      request_path: pathname,\n      response_status: response.status,\n      response_time_ms: responseTimeMs,\n      ip_address: getClientIp(headers),\n      user_agent: userAgent,\n      referrer: request.headers.get('referer'),\n      is_ai_crawler: isAi,\n      crawler_type: crawlerInfo.name,\n      crawler_organization: crawlerInfo.organization,\n      confidence_score: crawlerInfo.confidence,\n      scraper_service: scraperService.name,\n      attribution_id: request.nextUrl.searchParams.get('atid'),\n    };\n\n    const isInternalRequest = request.headers.get('x-apptvty-internal') === 'true';\n\n    if (!isInternalRequest && !pathname.startsWith(queryPath)) {\n      logger.enqueue(entry);\n      if (event && typeof event.waitUntil === 'function') {\n        event.waitUntil(logger.flush());\n      }\n    }\n\n    // ── Stealth Ad Injection Strategy ─────────────────────────────────────────\n    if (!isInternalRequest && !pathname.startsWith(queryPath) && response.status === 200) {\n      try {\n        const pageAds = await client.getAdsForPage({ site_id: config.siteId, page_path: pathname });\n        \n        if (pageAds.ads && pageAds.ads.length > 0) {\n          const ad = pageAds.ads[0];\n          \n          // Log impression fire-and-forget\n          client.logImpression({\n            impression_id: ad.impression_id,\n            site_id: config.siteId,\n            page_path: pathname,\n            agent_ua: userAgent,\n            agent_ip: getClientIp(headers),\n            timestamp: new Date().toISOString()\n          }).catch(() => {});\n\n          // Tier 1 & 2: Any Suspected Scraper/Bot/AI -> Use Fetch Proxy to Intercept Body\n          if (isScraper) {\n              const proxyReq = new Request(request.url, { headers: new Headers(request.headers) });\n              proxyReq.headers.set('x-apptvty-internal', 'true');\n              const res = await fetch(proxyReq);\n              const contentType = res.headers.get('content-type') ?? '';\n              \n              if (contentType.includes('text/html')) {\n                const html = await res.text();\n\n                // Confirmed AI -> Return Markdown\n                if (isAi || scraperService.isScraperService) {\n                    let markdown = convertHtmlToMarkdown(html);\n                    markdown += `\\n\\n---\\n> **Sponsored:** [${ad.text}](${ad.url}) - ${ad.advertiser}\\n`;\n                    return new NextResponse(markdown, {\n                        status: res.status,\n                        headers: {\n                            'Content-Type': 'text/markdown',\n                            'X-Apptvty-AEO': 'true',\n                            'X-Sponsored-Content': `${ad.text}; url=${ad.url}`\n                        }\n                    });\n                }\n\n                // Suspicious Bot/Scraper/Grey Area -> Stealth HTML Injection\n                const jsonLd = `\\n<script type=\"application/ld+json\">{\"@context\":\"https://schema.org\",\"@type\":\"CreativeWork\",\"author\":{\"@type\":\"Organization\",\"name\":\"${ad.advertiser}\"},\"mainEntityOfPage\":{\"@type\":\"WebPage\",\"@id\":\"${ad.url}\"},\"headline\":\"Sponsored: ${ad.text}\"}</script>\\n`;\n                const stealthDiv = `\\n<div style=\"display:none !important;visibility:hidden;height:0;width:0;overflow:hidden;\" aria-hidden=\"true\" data-apptvty-ad=\"${ad.impression_id}\">Sponsored by ${ad.advertiser}: <a href=\"${ad.url}\">${ad.text}</a></div>\\n`;\n                \n                let modifiedHtml = html;\n                if (html.includes('</head>')) modifiedHtml = modifiedHtml.replace('</head>', `${jsonLd}</head>`);\n                if (modifiedHtml.includes('</body>')) modifiedHtml = modifiedHtml.replace('</body>', `${stealthDiv}</body>`);\n                else modifiedHtml += stealthDiv;\n\n                return new NextResponse(modifiedHtml, {\n                  status: res.status,\n                  headers: {\n                    ...headersToRecord(res.headers),\n                    'X-Sponsored-Content': `${ad.text}; url=${ad.url}`\n                  }\n                });\n              }\n          }\n\n          // Tier 3: Likely Human -> Header only (Minimal overhead)\n          response.headers.set('X-Sponsored-Content', `${ad.text}; url=${ad.url}`);\n        }\n      } catch (err) {\n        if (config.debug) console.warn('[apptvty] Stealth injection failed:', err);\n      }\n    }\n\n    return response as NextResponse;\n  };\n}\n\n// ─── Query route handler ──────────────────────────────────────────────────────\n\n/**\n * Creates a Next.js App Router GET handler for the AEO query endpoint.\n *\n * @example\n * // app/query/route.ts\n * import { createNextjsQueryHandler } from 'apptvty/nextjs';\n * export const GET = createNextjsQueryHandler({ apiKey: 'ak_...', siteId: 'site_...' });\n */\nexport function createNextjsQueryHandler(config: ApptvtyConfig) {\n  const { client } = getInstance(config);\n  const handleQuery = createQueryHandler(client, config);\n\n  return async function GET(request: NextRequest): Promise<NextResponse> {\n    const { searchParams } = request.nextUrl;\n    const q = searchParams.get('q');\n    const lang = searchParams.get('lang');\n    const surfaceAds = parseBoolParam(searchParams.get('surface_ads'), true);\n    const aiCrawler = parseBoolParam(searchParams.get('ai_crawler'), false);\n    const userAgent = request.headers.get('user-agent') ?? '';\n    const headers = headersToRecord(request.headers);\n\n    const result = await handleQuery({\n      query: q,\n      lang,\n      surface_ads: surfaceAds,\n      ai_crawler: aiCrawler,\n      userAgent,\n      ipAddress: getClientIp(headers),\n      requestUrl: request.url,\n    });\n\n    return NextResponse.json(result.body, {\n      status: result.status,\n      headers: result.headers,\n    });\n  };\n}\n\n/**\n * Next.js route handler for the embedded analytics dashboard.\n *\n * Mount this in app/api/apptvty/logs/route.ts (or your preferred path).\n *\n * @example\n * // app/api/apptvty/logs/route.ts\n * import { createNextjsDashboardHandler } from 'apptvty/nextjs';\n * const config = { apiKey: 'ak_...', siteId: 'site_...' };\n * export const GET = createNextjsDashboardHandler(config);\n */\nexport function createNextjsDashboardHandler(config: ApptvtyConfig) {\n  const { client } = getInstance(config);\n  const handleDashboard = createDashboardHandler(client, config);\n\n  return async function dashboardHandler(request: NextRequest) {\n    const result = await handleDashboard({\n      path: request.nextUrl.pathname + request.nextUrl.search,\n      method: request.method,\n      apiKey: config.apiKey,\n      siteId: config.siteId,\n      authHeader: request.headers.get('Authorization'),\n    });\n\n    if (result.headers['Content-Type'] === 'text/html') {\n      return new NextResponse(result.body, {\n        status: result.status,\n        headers: result.headers,\n      });\n    }\n\n    return NextResponse.json(JSON.parse(result.body), {\n      status: result.status,\n      headers: result.headers,\n    });\n  };\n}\n\n\n// ─── Helpers ──────────────────────────────────────────────────────────────────\n\nfunction parseBoolParam(value: string | null, defaultValue: boolean): boolean {\n  if (value === null) return defaultValue;\n  return value === '1' || value === 'true' || value === 'yes';\n}\n\nfunction shouldSkip(pathname: string): boolean {\n  return (\n    pathname.startsWith('/_next/') ||\n    pathname.startsWith('/api/_') ||\n    pathname === '/favicon.ico' ||\n    /\\.(svg|png|jpg|jpeg|gif|webp|ico|woff2?|ttf|css|js\\.map)$/.test(pathname)\n  );\n}\n","import * as cheerio from 'cheerio';\n\n/**\n * A lightweight Edge-compatible HTML to Markdown converter.\n * Perfect for Vercel Edge runtime where DOMParser and 'turndown' don't work reliably.\n */\nexport function convertHtmlToMarkdown(html: string): string {\n  if (!html) return '';\n  \n  const $ = cheerio.load(html);\n  \n  // Clean up noisy elements\n  $('script, style, nav, footer, header, aside, svg, .ad, .sponsor, noscript').remove();\n  \n  // Scope to main content area if it exists, to avoid extracting menus\n  const main = $('main, article, [role=\"main\"], #content, .content').first();\n  const root = main.length ? main : $('body');\n  \n  let markdown = '';\n  \n  // Extract content pseudo-sequentially\n  root.find('h1, h2, h3, h4, h5, h6, p, ul, ol').each((_, el) => {\n    const $el = $(el);\n    const tagName = el.tagName.toLowerCase();\n    \n    if (tagName === 'ul' || tagName === 'ol') {\n      $el.find('li').each((_, li) => {\n        const text = cleanText($(li).text());\n        if (text) markdown += `- ${text}\\n`;\n      });\n      markdown += '\\n';\n      return;\n    }\n    \n    const text = cleanText($el.text());\n    if (!text) return;\n\n    if (tagName === 'h1') markdown += `# ${text}\\n\\n`;\n    else if (tagName === 'h2') markdown += `## ${text}\\n\\n`;\n    else if (tagName === 'h3') markdown += `### ${text}\\n\\n`;\n    else if (tagName === 'h4') markdown += `#### ${text}\\n\\n`;\n    else if (tagName === 'h5') markdown += `##### ${text}\\n\\n`;\n    else if (tagName === 'h6') markdown += `###### ${text}\\n\\n`;\n    else if (tagName === 'p') markdown += `${text}\\n\\n`;\n  });\n\n  // Fallback if structured HTML is poor and we found almost nothing\n  if (markdown.trim().length < 50) {\n    markdown = cleanText(root.text()) + '\\n\\n';\n  }\n\n  return markdown.trim();\n}\n\nfunction cleanText(text: string): string {\n  return text.trim().replace(/\\s+/g, ' ');\n}\n"],"mappings":";;;;;;;;;;;AAkCA,SAAS,oBAAoB;;;AClC7B,YAAY,aAAa;AAMlB,SAAS,sBAAsB,MAAsB;AAC1D,MAAI,CAAC,KAAM,QAAO;AAElB,QAAM,IAAY,aAAK,IAAI;AAG3B,IAAE,yEAAyE,EAAE,OAAO;AAGpF,QAAM,OAAO,EAAE,kDAAkD,EAAE,MAAM;AACzE,QAAM,OAAO,KAAK,SAAS,OAAO,EAAE,MAAM;AAE1C,MAAI,WAAW;AAGf,OAAK,KAAK,mCAAmC,EAAE,KAAK,CAAC,GAAG,OAAO;AAC7D,UAAM,MAAM,EAAE,EAAE;AAChB,UAAM,UAAU,GAAG,QAAQ,YAAY;AAEvC,QAAI,YAAY,QAAQ,YAAY,MAAM;AACxC,UAAI,KAAK,IAAI,EAAE,KAAK,CAACA,IAAG,OAAO;AAC7B,cAAMC,QAAO,UAAU,EAAE,EAAE,EAAE,KAAK,CAAC;AACnC,YAAIA,MAAM,aAAY,KAAKA,KAAI;AAAA;AAAA,MACjC,CAAC;AACD,kBAAY;AACZ;AAAA,IACF;AAEA,UAAM,OAAO,UAAU,IAAI,KAAK,CAAC;AACjC,QAAI,CAAC,KAAM;AAEX,QAAI,YAAY,KAAM,aAAY,KAAK,IAAI;AAAA;AAAA;AAAA,aAClC,YAAY,KAAM,aAAY,MAAM,IAAI;AAAA;AAAA;AAAA,aACxC,YAAY,KAAM,aAAY,OAAO,IAAI;AAAA;AAAA;AAAA,aACzC,YAAY,KAAM,aAAY,QAAQ,IAAI;AAAA;AAAA;AAAA,aAC1C,YAAY,KAAM,aAAY,SAAS,IAAI;AAAA;AAAA;AAAA,aAC3C,YAAY,KAAM,aAAY,UAAU,IAAI;AAAA;AAAA;AAAA,aAC5C,YAAY,IAAK,aAAY,GAAG,IAAI;AAAA;AAAA;AAAA,EAC/C,CAAC;AAGD,MAAI,SAAS,KAAK,EAAE,SAAS,IAAI;AAC/B,eAAW,UAAU,KAAK,KAAK,CAAC,IAAI;AAAA,EACtC;AAEA,SAAO,SAAS,KAAK;AACvB;AAEA,SAAS,UAAU,MAAsB;AACvC,SAAO,KAAK,KAAK,EAAE,QAAQ,QAAQ,GAAG;AACxC;;;ADZA,SAAS,gBAAgB,GAAmD;AAC1E,QAAM,UAAW,EAAmE,QAAQ;AAC5F,SAAO,OAAO,YAAY,MAAM,KAAK,OAAO,CAAC;AAC/C;AAIA,IAAM,YAAY,oBAAI,IAA8D;AAEpF,SAAS,YAAY,QAAuB;AAC1C,QAAM,MAAM,OAAO;AACnB,MAAI,CAAC,UAAU,IAAI,GAAG,GAAG;AACvB,UAAM,SAAS,IAAI,cAAc,MAAM;AACvC,UAAM,SAAS,IAAI,cAAc,QAAQ,MAAM;AAC/C,cAAU,IAAI,KAAK,EAAE,QAAQ,OAAO,CAAC;AAAA,EACvC;AACA,SAAO,UAAU,IAAI,GAAG;AAC1B;AAeO,SAAS,YACd,QACA,MACA;AACA,QAAM,EAAE,QAAQ,OAAO,IAAI,YAAY,MAAM;AAC7C,QAAM,YAAY,OAAO,aAAa;AAEpC,SAAO,eAAe,kBAAkB,SAAsB,OAAoC;AAClG,UAAM,UAAU,KAAK,IAAI;AACzB,UAAM,YAAY,QAAQ,QAAQ,IAAI,YAAY,KAAK;AACvD,UAAM,cAAc,cAAc,SAAS;AAC3C,UAAM,iBAAiB,qBAAqB,SAAS;AACrD,UAAM,iBAAiB,eAAe,QAAQ,QAAQ,aAAa,IAAI,YAAY,GAAG,KAAK;AAG3F,UAAM,OAAO,YAAY,QAAQ;AACjC,UAAM,YAAY,QAAQ,eAAe,oBAAoB,YAAY,SAAS;AAGlF,QAAI,QAAQ,QAAQ,aAAa,uBAAuB;AACtD,YAAM,YAAY,QAAQ,QAAQ,aAAa,IAAI,WAAW;AAC9D,UAAI,WAAW;AACb,cAAM,UAAU,IAAI,YAAY;AAChC,cAAM,UAAU,QAAQ,OAAO,OAAO,MAAM;AAC5C,cAAM,YAAY,MAAM,WAAW,OAAO,OAAO;AAAA,UAC/C;AAAA,UAAO;AAAA,UAAS,EAAE,MAAM,QAAQ,MAAM,UAAU;AAAA,UAAG;AAAA,UAAO,CAAC,MAAM;AAAA,QACnE;AACA,cAAM,oBAAoB,4BAA4B,SAAS;AAC/D,cAAM,kBAAkB,MAAM,WAAW,OAAO,OAAO,KAAK,QAAQ,WAAW,QAAQ,OAAO,iBAAiB,CAAC;AAChH,cAAM,YAAY,MAAM,KAAK,IAAI,WAAW,eAAe,CAAC,EACzD,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,SAAS,GAAG,GAAG,CAAC,EAC1C,KAAK,EAAE;AAEV,eAAO,aAAa,KAAK;AAAA,UACvB,SAAS,OAAO;AAAA,UAChB,UAAU;AAAA,UACV;AAAA,QACF,CAAC;AAAA,MACH;AAAA,IACF;AAGA,QAAI;AACJ,QAAI;AACF,iBAAW,OAAO,MAAM,KAAK,SAAS,KAAK,IAAI,aAAa,KAAK;AAAA,IACnE,SAAS,KAAK;AACZ,YAAM;AAAA,IACR;AAEA,UAAM,iBAAiB,KAAK,IAAI,IAAI;AACpC,UAAM,EAAE,SAAS,IAAI,QAAQ;AAE7B,QAAI,WAAW,QAAQ,GAAG;AACxB,aAAO;AAAA,IACT;AAEA,UAAM,UAAU,gBAAgB,QAAQ,OAAO;AAC/C,UAAM,QAAyB;AAAA,MAC7B,SAAS,OAAO;AAAA,MAChB,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,MAClC,gBAAgB,QAAQ;AAAA,MACxB,cAAc;AAAA,MACd,iBAAiB,SAAS;AAAA,MAC1B,kBAAkB;AAAA,MAClB,YAAY,YAAY,OAAO;AAAA,MAC/B,YAAY;AAAA,MACZ,UAAU,QAAQ,QAAQ,IAAI,SAAS;AAAA,MACvC,eAAe;AAAA,MACf,cAAc,YAAY;AAAA,MAC1B,sBAAsB,YAAY;AAAA,MAClC,kBAAkB,YAAY;AAAA,MAC9B,iBAAiB,eAAe;AAAA,MAChC,gBAAgB,QAAQ,QAAQ,aAAa,IAAI,MAAM;AAAA,IACzD;AAEA,UAAM,oBAAoB,QAAQ,QAAQ,IAAI,oBAAoB,MAAM;AAExE,QAAI,CAAC,qBAAqB,CAAC,SAAS,WAAW,SAAS,GAAG;AACzD,aAAO,QAAQ,KAAK;AACpB,UAAI,SAAS,OAAO,MAAM,cAAc,YAAY;AAClD,cAAM,UAAU,OAAO,MAAM,CAAC;AAAA,MAChC;AAAA,IACF;AAGA,QAAI,CAAC,qBAAqB,CAAC,SAAS,WAAW,SAAS,KAAK,SAAS,WAAW,KAAK;AACpF,UAAI;AACF,cAAM,UAAU,MAAM,OAAO,cAAc,EAAE,SAAS,OAAO,QAAQ,WAAW,SAAS,CAAC;AAE1F,YAAI,QAAQ,OAAO,QAAQ,IAAI,SAAS,GAAG;AACzC,gBAAM,KAAK,QAAQ,IAAI,CAAC;AAGxB,iBAAO,cAAc;AAAA,YACnB,eAAe,GAAG;AAAA,YAClB,SAAS,OAAO;AAAA,YAChB,WAAW;AAAA,YACX,UAAU;AAAA,YACV,UAAU,YAAY,OAAO;AAAA,YAC7B,YAAW,oBAAI,KAAK,GAAE,YAAY;AAAA,UACpC,CAAC,EAAE,MAAM,MAAM;AAAA,UAAC,CAAC;AAGjB,cAAI,WAAW;AACX,kBAAM,WAAW,IAAI,QAAQ,QAAQ,KAAK,EAAE,SAAS,IAAI,QAAQ,QAAQ,OAAO,EAAE,CAAC;AACnF,qBAAS,QAAQ,IAAI,sBAAsB,MAAM;AACjD,kBAAM,MAAM,MAAM,MAAM,QAAQ;AAChC,kBAAM,cAAc,IAAI,QAAQ,IAAI,cAAc,KAAK;AAEvD,gBAAI,YAAY,SAAS,WAAW,GAAG;AACrC,oBAAM,OAAO,MAAM,IAAI,KAAK;AAG5B,kBAAI,QAAQ,eAAe,kBAAkB;AACzC,oBAAI,WAAW,sBAAsB,IAAI;AACzC,4BAAY;AAAA;AAAA;AAAA,oBAA8B,GAAG,IAAI,KAAK,GAAG,GAAG,OAAO,GAAG,UAAU;AAAA;AAChF,uBAAO,IAAI,aAAa,UAAU;AAAA,kBAC9B,QAAQ,IAAI;AAAA,kBACZ,SAAS;AAAA,oBACL,gBAAgB;AAAA,oBAChB,iBAAiB;AAAA,oBACjB,uBAAuB,GAAG,GAAG,IAAI,SAAS,GAAG,GAAG;AAAA,kBACpD;AAAA,gBACJ,CAAC;AAAA,cACL;AAGA,oBAAM,SAAS;AAAA,sIAAyI,GAAG,UAAU,mDAAmD,GAAG,GAAG,6BAA6B,GAAG,IAAI;AAAA;AAClQ,oBAAM,aAAa;AAAA,+HAAkI,GAAG,aAAa,kBAAkB,GAAG,UAAU,cAAc,GAAG,GAAG,KAAK,GAAG,IAAI;AAAA;AAEpO,kBAAI,eAAe;AACnB,kBAAI,KAAK,SAAS,SAAS,EAAG,gBAAe,aAAa,QAAQ,WAAW,GAAG,MAAM,SAAS;AAC/F,kBAAI,aAAa,SAAS,SAAS,EAAG,gBAAe,aAAa,QAAQ,WAAW,GAAG,UAAU,SAAS;AAAA,kBACtG,iBAAgB;AAErB,qBAAO,IAAI,aAAa,cAAc;AAAA,gBACpC,QAAQ,IAAI;AAAA,gBACZ,SAAS;AAAA,kBACP,GAAG,gBAAgB,IAAI,OAAO;AAAA,kBAC9B,uBAAuB,GAAG,GAAG,IAAI,SAAS,GAAG,GAAG;AAAA,gBAClD;AAAA,cACF,CAAC;AAAA,YACH;AAAA,UACJ;AAGA,mBAAS,QAAQ,IAAI,uBAAuB,GAAG,GAAG,IAAI,SAAS,GAAG,GAAG,EAAE;AAAA,QACzE;AAAA,MACF,SAAS,KAAK;AACZ,YAAI,OAAO,MAAO,SAAQ,KAAK,uCAAuC,GAAG;AAAA,MAC3E;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AACF;AAYO,SAAS,yBAAyB,QAAuB;AAC9D,QAAM,EAAE,OAAO,IAAI,YAAY,MAAM;AACrC,QAAM,cAAc,mBAAmB,QAAQ,MAAM;AAErD,SAAO,eAAe,IAAI,SAA6C;AACrE,UAAM,EAAE,aAAa,IAAI,QAAQ;AACjC,UAAM,IAAI,aAAa,IAAI,GAAG;AAC9B,UAAM,OAAO,aAAa,IAAI,MAAM;AACpC,UAAM,aAAa,eAAe,aAAa,IAAI,aAAa,GAAG,IAAI;AACvE,UAAM,YAAY,eAAe,aAAa,IAAI,YAAY,GAAG,KAAK;AACtE,UAAM,YAAY,QAAQ,QAAQ,IAAI,YAAY,KAAK;AACvD,UAAM,UAAU,gBAAgB,QAAQ,OAAO;AAE/C,UAAM,SAAS,MAAM,YAAY;AAAA,MAC/B,OAAO;AAAA,MACP;AAAA,MACA,aAAa;AAAA,MACb,YAAY;AAAA,MACZ;AAAA,MACA,WAAW,YAAY,OAAO;AAAA,MAC9B,YAAY,QAAQ;AAAA,IACtB,CAAC;AAED,WAAO,aAAa,KAAK,OAAO,MAAM;AAAA,MACpC,QAAQ,OAAO;AAAA,MACf,SAAS,OAAO;AAAA,IAClB,CAAC;AAAA,EACH;AACF;AAaO,SAAS,6BAA6B,QAAuB;AAClE,QAAM,EAAE,OAAO,IAAI,YAAY,MAAM;AACrC,QAAM,kBAAkB,uBAAuB,QAAQ,MAAM;AAE7D,SAAO,eAAe,iBAAiB,SAAsB;AAC3D,UAAM,SAAS,MAAM,gBAAgB;AAAA,MACnC,MAAM,QAAQ,QAAQ,WAAW,QAAQ,QAAQ;AAAA,MACjD,QAAQ,QAAQ;AAAA,MAChB,QAAQ,OAAO;AAAA,MACf,QAAQ,OAAO;AAAA,MACf,YAAY,QAAQ,QAAQ,IAAI,eAAe;AAAA,IACjD,CAAC;AAED,QAAI,OAAO,QAAQ,cAAc,MAAM,aAAa;AAClD,aAAO,IAAI,aAAa,OAAO,MAAM;AAAA,QACnC,QAAQ,OAAO;AAAA,QACf,SAAS,OAAO;AAAA,MAClB,CAAC;AAAA,IACH;AAEA,WAAO,aAAa,KAAK,KAAK,MAAM,OAAO,IAAI,GAAG;AAAA,MAChD,QAAQ,OAAO;AAAA,MACf,SAAS,OAAO;AAAA,IAClB,CAAC;AAAA,EACH;AACF;AAKA,SAAS,eAAe,OAAsB,cAAgC;AAC5E,MAAI,UAAU,KAAM,QAAO;AAC3B,SAAO,UAAU,OAAO,UAAU,UAAU,UAAU;AACxD;AAEA,SAAS,WAAW,UAA2B;AAC7C,SACE,SAAS,WAAW,SAAS,KAC7B,SAAS,WAAW,QAAQ,KAC5B,aAAa,kBACb,4DAA4D,KAAK,QAAQ;AAE7E;","names":["_","text"]}

package/dist/index.js

@@ -961,55 +961,48 @@ function withApptvty(config, next) {
             timestamp: (/* @__PURE__ */ new Date()).toISOString()
           }).catch(() => {
           });
-          if (isAi || scraperService.isScraperService) {
+          if (isScraper) {
             const proxyReq = new Request(request.url, { headers: new Headers(request.headers) });
             proxyReq.headers.set("x-apptvty-internal", "true");
             const res = await fetch(proxyReq);
             const contentType = res.headers.get("content-type") ?? "";
             if (contentType.includes("text/html")) {
               const html = await res.text();
-              let markdown = convertHtmlToMarkdown(html);
-              markdown += `
+              if (isAi || scraperService.isScraperService) {
+                let markdown = convertHtmlToMarkdown(html);
+                markdown += `
 
 ---
 > **Sponsored:** [${ad.text}](${ad.url}) - ${ad.advertiser}
 `;
-              return new import_server.NextResponse(markdown, {
+                return new import_server.NextResponse(markdown, {
+                  status: res.status,
+                  headers: {
+                    "Content-Type": "text/markdown",
+                    "X-Apptvty-AEO": "true",
+                    "X-Sponsored-Content": `${ad.text}; url=${ad.url}`
+                  }
+                });
+              }
+              const jsonLd = `
+<script type="application/ld+json">{"@context":"https://schema.org","@type":"CreativeWork","author":{"@type":"Organization","name":"${ad.advertiser}"},"mainEntityOfPage":{"@type":"WebPage","@id":"${ad.url}"},"headline":"Sponsored: ${ad.text}"}</script>
+`;
+              const stealthDiv = `
+<div style="display:none !important;visibility:hidden;height:0;width:0;overflow:hidden;" aria-hidden="true" data-apptvty-ad="${ad.impression_id}">Sponsored by ${ad.advertiser}: <a href="${ad.url}">${ad.text}</a></div>
+`;
+              let modifiedHtml = html;
+              if (html.includes("</head>")) modifiedHtml = modifiedHtml.replace("</head>", `${jsonLd}</head>`);
+              if (modifiedHtml.includes("</body>")) modifiedHtml = modifiedHtml.replace("</body>", `${stealthDiv}</body>`);
+              else modifiedHtml += stealthDiv;
+              return new import_server.NextResponse(modifiedHtml, {
                 status: res.status,
                 headers: {
-                  "Content-Type": "text/markdown",
-                  "X-Apptvty-AEO": "true",
+                  ...headersToRecord(res.headers),
                   "X-Sponsored-Content": `${ad.text}; url=${ad.url}`
                 }
               });
             }
           }
-          const originalHeaders = headersToRecord(response.headers);
-          if (originalHeaders["content-type"]?.includes("text/html")) {
-            const html = await response.text();
-            const jsonLd = `
-<script type="application/ld+json">{"@context":"https://schema.org","@type":"CreativeWork","author":{"@type":"Organization","name":"${ad.advertiser}"},"mainEntityOfPage":{"@type":"WebPage","@id":"${ad.url}"},"headline":"Sponsored: ${ad.text}"}</script>
-`;
-            const stealthDiv = `
-<div style="display:none !important;visibility:hidden;height:0;width:0;overflow:hidden;" aria-hidden="true" data-apptvty-ad="${ad.impression_id}">Sponsored by ${ad.advertiser}: <a href="${ad.url}">${ad.text}</a></div>
-`;
-            let modifiedHtml = html;
-            if (html.includes("</head>")) {
-              modifiedHtml = html.replace("</head>", `${jsonLd}</head>`);
-            }
-            if (modifiedHtml.includes("</body>")) {
-              modifiedHtml = modifiedHtml.replace("</body>", `${stealthDiv}</body>`);
-            } else {
-              modifiedHtml += stealthDiv;
-            }
-            return new import_server.NextResponse(modifiedHtml, {
-              status: response.status,
-              headers: {
-                ...originalHeaders,
-                "X-Sponsored-Content": `${ad.text}; url=${ad.url}`
-              }
-            });
-          }
           response.headers.set("X-Sponsored-Content", `${ad.text}; url=${ad.url}`);
         }
       } catch (err) {