apptvty 0.2.5 → 0.3.1

package/dist/index.js

@@ -1,7 +1,9 @@
 "use strict";
+var __create = Object.create;
 var __defProp = Object.defineProperty;
 var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
 var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
 var __hasOwnProp = Object.prototype.hasOwnProperty;
 var __export = (target, all) => {
   for (var name in all)
@@ -15,6 +17,14 @@ var __copyProps = (to, from, except, desc) => {
   }
   return to;
 };
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
 var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
 
 // src/index.ts
@@ -361,7 +371,7 @@ var ApptvtyClient = class {
           this.log("X402 challenge auto-paid from wallet, retrying request");
           return this.post(path, body);
         }
-        let dashboardUrl = "https://dashboard.apptvty.com/login";
+        let dashboardUrl = (typeof process !== "undefined" ? process.env?.DASHBOARD_URL : void 0) ?? "https://dashboard.apptvty.com/login";
         try {
           const json = JSON.parse(text);
           dashboardUrl = json?.error?.details?.dashboard_url ?? dashboardUrl;
@@ -413,7 +423,7 @@ var RequestLogger = class {
     this.client = client;
     this.queue = [];
     this.timer = null;
-    this.flushing = false;
+    this.activeFlushPromise = null;
     this.batchSize = config.batchSize ?? 50;
     this.debug = config.debug ?? false;
     const interval = config.flushInterval ?? 5e3;
@@ -438,21 +448,31 @@ var RequestLogger = class {
   /** Enqueue a single log entry. Non-blocking. */
   enqueue(entry) {
     this.queue.push(entry);
-    if (this.queue.length >= this.batchSize) {
-      void this.flush();
-    }
+    void this.flush();
   }
-  /** Flush the current queue to the API. */
+  /** Flush the current queue to the API. Returns the active Promise so Edge runtimes can wait. */
   async flush() {
-    if (this.flushing || this.queue.length === 0) return;
-    this.flushing = true;
-    const batch = this.queue.splice(0, this.batchSize);
-    try {
-      await this.client.sendLogs(batch);
-    } catch {
-    } finally {
-      this.flushing = false;
+    if (this.queue.length === 0) {
+      return this.activeFlushPromise || Promise.resolve();
+    }
+    if (this.activeFlushPromise) {
+      await this.activeFlushPromise;
+      if (this.queue.length > 0) return this.flush();
+      return Promise.resolve();
     }
+    const batch = this.queue.splice(0, this.batchSize);
+    this.activeFlushPromise = (async () => {
+      try {
+        await this.client.sendLogs(batch);
+      } catch {
+      } finally {
+        this.activeFlushPromise = null;
+        if (this.queue.length > 0) {
+          void this.flush();
+        }
+      }
+    })();
+    return this.activeFlushPromise;
   }
   /**
    * Synchronous-ish flush for process shutdown.
@@ -676,7 +696,7 @@ function createQueryHandler(client, config) {
     if (trimmedQuery.length > 500) {
       return errorResponse(400, "QUERY_TOO_LONG", "Query must be 500 characters or fewer");
     }
-    const requestId = crypto.randomUUID();
+    const requestId = globalThis.crypto.randomUUID();
     const timestamp = (/* @__PURE__ */ new Date()).toISOString();
     const startMs = Date.now();
     const surfaceAds = req.surface_ads !== false;
@@ -704,16 +724,21 @@ function createQueryHandler(client, config) {
       return errorResponse(502, "UPSTREAM_ERROR", "Could not retrieve an answer at this time");
     }
     const responseTimeMs = Date.now() - startMs;
-    const ads = backendResponse.sponsored ? Array.isArray(backendResponse.sponsored) ? backendResponse.sponsored : [backendResponse.sponsored] : [];
-    for (const ad of ads) {
-      void client.logImpression({
-        impression_id: ad.impression_id,
-        site_id: config.siteId,
-        query: trimmedQuery,
-        agent_ua: req.userAgent,
-        agent_ip: req.ipAddress,
-        timestamp
-      });
+    const ads = backendResponse.related_resources ? Array.isArray(backendResponse.related_resources) ? backendResponse.related_resources : [backendResponse.related_resources] : [];
+    if (ads.length > 0) {
+      await Promise.all(
+        ads.map(
+          (ad) => client.logImpression({
+            impression_id: ad.impression_id,
+            site_id: config.siteId,
+            query: trimmedQuery,
+            agent_ua: req.userAgent,
+            agent_ip: req.ipAddress,
+            timestamp
+          }).catch(() => {
+          })
+        )
+      );
     }
     const agentResponse = {
       success: true,
@@ -722,7 +747,7 @@ function createQueryHandler(client, config) {
       answer: backendResponse.answer,
       sources: backendResponse.sources,
       confidence: backendResponse.confidence,
-      ...backendResponse.sponsored && { sponsored: backendResponse.sponsored },
+      ...backendResponse.related_resources && { related_resources: backendResponse.related_resources },
       metadata: {
         request_id: requestId,
         response_time_ms: responseTimeMs,
@@ -740,7 +765,7 @@ function errorResponse(status, code, message) {
     error: {
       code,
       message,
-      request_id: crypto.randomUUID(),
+      request_id: globalThis.crypto.randomUUID(),
       timestamp: (/* @__PURE__ */ new Date()).toISOString()
     }
   };
@@ -758,61 +783,58 @@ function getOrigin(url) {
 // src/middleware/nextjs.ts
 var import_server = require("next/server");
 
-// src/ad-injection.ts
-var AD_INJECTION_MARKER = "<!-- apptvty-sponsored -->";
-function injectIntoHtml(html, ads, isScraperService) {
-  if (!html || ads.length === 0) return html;
-  if (html.includes(AD_INJECTION_MARKER)) return html;
-  let modified = html;
-  const contentBlock = buildContentStreamBlock(ads);
-  if (modified.includes("</article>")) {
-    modified = modified.replace("</article>", `${contentBlock}
-</article>`);
-  } else if (modified.includes("</main>")) {
-    modified = modified.replace("</main>", `${contentBlock}
-</main>`);
-  } else if (!isScraperService && modified.includes("</body>")) {
-    modified = modified.replace("</body>", `${contentBlock}
-</body>`);
-  }
-  if (!isScraperService && modified.includes("</head>")) {
-    const jsonLdBlock = buildJsonLdBlock(ads);
-    modified = modified.replace("</head>", `${jsonLdBlock}
-</head>`);
+// src/markdown.ts
+var cheerio = __toESM(require("cheerio"));
+function convertHtmlToMarkdown(html) {
+  if (!html) return "";
+  const $ = cheerio.load(html);
+  $("script, style, nav, footer, header, aside, svg, .ad, .sponsor, noscript").remove();
+  const main = $('main, article, [role="main"], #content, .content').first();
+  const root = main.length ? main : $("body");
+  let markdown = "";
+  root.find("h1, h2, h3, h4, h5, h6, p, ul, ol").each((_, el) => {
+    const $el = $(el);
+    const tagName = el.tagName.toLowerCase();
+    if (tagName === "ul" || tagName === "ol") {
+      $el.find("li").each((_2, li) => {
+        const text2 = cleanText($(li).text());
+        if (text2) markdown += `- ${text2}
+`;
+      });
+      markdown += "\n";
+      return;
+    }
+    const text = cleanText($el.text());
+    if (!text) return;
+    if (tagName === "h1") markdown += `# ${text}
+
+`;
+    else if (tagName === "h2") markdown += `## ${text}
+
+`;
+    else if (tagName === "h3") markdown += `### ${text}
+
+`;
+    else if (tagName === "h4") markdown += `#### ${text}
+
+`;
+    else if (tagName === "h5") markdown += `##### ${text}
+
+`;
+    else if (tagName === "h6") markdown += `###### ${text}
+
+`;
+    else if (tagName === "p") markdown += `${text}
+
+`;
+  });
+  if (markdown.trim().length < 50) {
+    markdown = cleanText(root.text()) + "\n\n";
   }
-  return modified;
-}
-function buildSponsoredHeader(ads) {
-  return JSON.stringify(
-    ads.map((ad) => ({ text: ad.text, url: ad.url, advertiser: ad.advertiser }))
-  );
-}
-function buildContentStreamBlock(ads) {
-  const paragraphs = ads.map(
-    (ad) => `<p data-apptvty-sponsored="${escapeAttr(ad.impression_id)}"><strong>[Sponsored]</strong> <a href="${escapeAttr(ad.url)}" rel="sponsored noopener">${escapeHtml(ad.text)}</a> \u2014 <span>${escapeHtml(ad.advertiser)}</span></p>`
-  ).join("\n");
-  return `${AD_INJECTION_MARKER}
-${paragraphs}`;
+  return markdown.trim();
 }
-function buildJsonLdBlock(ads) {
-  const entries = ads.map((ad) => ({
-    "@context": "https://schema.org",
-    "@type": "WPAdBlock",
-    sponsor: {
-      "@type": "Organization",
-      name: ad.advertiser,
-      url: ad.url
-    },
-    description: ad.text
-  }));
-  const ld = entries.length === 1 ? entries[0] : entries;
-  return `<script type="application/ld+json">${JSON.stringify(ld)}</script>`;
-}
-function escapeHtml(s) {
-  return s.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;").replace(/'/g, "&#39;");
-}
-function escapeAttr(s) {
-  return s.replace(/"/g, "&quot;").replace(/'/g, "&#39;");
+function cleanText(text) {
+  return text.trim().replace(/\s+/g, " ");
 }
 
 // src/middleware/nextjs.ts
@@ -833,16 +855,38 @@ function getInstance(config) {
 function withApptvty(config, next) {
   const { client, logger } = getInstance(config);
   const queryPath = config.queryPath ?? "/query";
-  return async function apptvtyMiddleware(request) {
+  return async function apptvtyMiddleware(request, event) {
     const startMs = Date.now();
     const userAgent = request.headers.get("user-agent") ?? "";
     const crawlerInfo = detectCrawler(userAgent);
     const scraperService = detectScraperService(userAgent);
     const aiCrawlerParam = parseBoolParam(request.nextUrl.searchParams.get("ai_crawler"), false);
     const isCrawler = crawlerInfo.isAi || aiCrawlerParam || scraperService.isScraperService;
+    if (request.nextUrl.pathname === "/api/apptvty/verify") {
+      const challenge = request.nextUrl.searchParams.get("challenge");
+      if (challenge) {
+        const encoder = new TextEncoder();
+        const keyData = encoder.encode(config.apiKey);
+        const cryptoKey = await globalThis.crypto.subtle.importKey(
+          "raw",
+          keyData,
+          { name: "HMAC", hash: "SHA-256" },
+          false,
+          ["sign"]
+        );
+        const prefixedChallenge = `apptvty_verify_challenge:${challenge}`;
+        const signatureBuffer = await globalThis.crypto.subtle.sign("HMAC", cryptoKey, encoder.encode(prefixedChallenge));
+        const signature = Array.from(new Uint8Array(signatureBuffer)).map((b) => b.toString(16).padStart(2, "0")).join("");
+        return import_server.NextResponse.json({
+          site_id: config.siteId,
+          verified: true,
+          signature
+        });
+      }
+    }
     let response;
     try {
-      response = next ? await next(request) : import_server.NextResponse.next();
+      response = next ? await next(request, event) : import_server.NextResponse.next();
     } catch (err) {
       throw err;
     }
@@ -868,24 +912,53 @@ function withApptvty(config, next) {
       confidence_score: crawlerInfo.confidence,
       scraper_service: scraperService.name
     };
-    logger.enqueue(entry);
-    if (isCrawler && response.ok && !pathname.startsWith(queryPath)) {
-      const contentType = response.headers.get("content-type") ?? "";
-      if (contentType.includes("text/html")) {
-        try {
-          const modified = await injectAdsIntoResponse(
-            response,
-            client,
-            config,
-            pathname,
-            userAgent,
-            getClientIp(headers),
-            scraperService.isScraperService
-          );
-          if (modified) return modified;
-        } catch (err) {
-          if (config.debug) console.warn("[apptvty] Ad injection failed:", err);
+    const isInternalRequest = request.headers.get("x-apptvty-internal") === "true";
+    if (!isInternalRequest && !pathname.startsWith(queryPath)) {
+      logger.enqueue(entry);
+      if (event && typeof event.waitUntil === "function") {
+        event.waitUntil(logger.flush());
+      }
+    }
+    if (isCrawler && !isInternalRequest && !pathname.startsWith(queryPath)) {
+      try {
+        const proxyReq = new Request(request.url, {
+          headers: new Headers(request.headers)
+        });
+        proxyReq.headers.set("x-apptvty-internal", "true");
+        const res = await fetch(proxyReq);
+        const contentType = res.headers.get("content-type") ?? "";
+        if (contentType.includes("text/html")) {
+          const html = await res.text();
+          let markdown = convertHtmlToMarkdown(html);
+          const pageAds = await client.getAdsForPage({ site_id: config.siteId, page_path: pathname });
+          if (pageAds.ads && pageAds.ads.length > 0) {
+            const ad = pageAds.ads[0];
+            markdown += `
+
+---
+> **Sponsored:** [${ad.text}](${ad.url}) - ${ad.advertiser}
+`;
+            client.logImpression({
+              impression_id: ad.impression_id,
+              site_id: config.siteId,
+              page_path: pathname,
+              agent_ua: userAgent,
+              agent_ip: getClientIp(headers),
+              timestamp: (/* @__PURE__ */ new Date()).toISOString()
+            }).catch(() => {
+            });
+          }
+          return new import_server.NextResponse(markdown, {
+            status: res.status,
+            headers: {
+              "Content-Type": "text/markdown",
+              "X-Apptvty-AEO": "true"
+            }
+          });
         }
+        return res;
+      } catch (err) {
+        if (config.debug) console.warn("[apptvty] Markdown proxy failed:", err);
       }
     }
     return response;
@@ -917,33 +990,6 @@ function createNextjsQueryHandler(config) {
     });
   };
 }
-async function injectAdsIntoResponse(response, client, config, pathname, userAgent, ipAddress, isScraperService) {
-  const html = await response.text();
-  if (!html) return null;
-  const pageAds = await client.getAdsForPage({ site_id: config.siteId, page_path: pathname });
-  if (!pageAds.ads || pageAds.ads.length === 0) return null;
-  const modified = injectIntoHtml(html, pageAds.ads, isScraperService);
-  if (modified === html) return null;
-  const newHeaders = new Headers(response.headers);
-  newHeaders.set("X-Sponsored-Content", buildSponsoredHeader(pageAds.ads));
-  const timestamp = (/* @__PURE__ */ new Date()).toISOString();
-  for (const ad of pageAds.ads) {
-    client.logImpression({
-      impression_id: ad.impression_id,
-      site_id: config.siteId,
-      page_path: pathname,
-      agent_ua: userAgent,
-      agent_ip: ipAddress,
-      timestamp
-    }).catch(() => {
-    });
-  }
-  return new import_server.NextResponse(modified, {
-    status: response.status,
-    statusText: response.statusText,
-    headers: newHeaders
-  });
-}
 function parseBoolParam(value, defaultValue) {
   if (value === null) return defaultValue;
   return value === "1" || value === "true" || value === "yes";
@@ -952,6 +998,63 @@ function shouldSkip(pathname) {
   return pathname.startsWith("/_next/") || pathname.startsWith("/api/_") || pathname === "/favicon.ico" || /\.(svg|png|jpg|jpeg|gif|webp|ico|woff2?|ttf|css|js\.map)$/.test(pathname);
 }
 
+// src/ad-injection.ts
+var AD_INJECTION_MARKER = "<!-- apptvty-sponsored -->";
+function injectIntoHtml(html, ads, isScraperService) {
+  if (!html || ads.length === 0) return html;
+  if (html.includes(AD_INJECTION_MARKER)) return html;
+  let modified = html;
+  const contentBlock = buildContentStreamBlock(ads);
+  if (modified.includes("</article>")) {
+    modified = modified.replace("</article>", `${contentBlock}
+</article>`);
+  } else if (modified.includes("</main>")) {
+    modified = modified.replace("</main>", `${contentBlock}
+</main>`);
+  } else if (!isScraperService && modified.includes("</body>")) {
+    modified = modified.replace("</body>", `${contentBlock}
+</body>`);
+  }
+  if (!isScraperService && modified.includes("</head>")) {
+    const jsonLdBlock = buildJsonLdBlock(ads);
+    modified = modified.replace("</head>", `${jsonLdBlock}
+</head>`);
+  }
+  return modified;
+}
+function buildSponsoredHeader(ads) {
+  return JSON.stringify(
+    ads.map((ad) => ({ text: ad.text, url: ad.url, advertiser: ad.advertiser }))
+  );
+}
+function buildContentStreamBlock(ads) {
+  const paragraphs = ads.map(
+    (ad) => `<p data-apptvty-sponsored="${escapeAttr(ad.impression_id)}"><strong>[Sponsored]</strong> <a href="${escapeAttr(ad.url)}" rel="sponsored noopener">${escapeHtml(ad.text)}</a> \u2014 <span>${escapeHtml(ad.advertiser)}</span></p>`
+  ).join("\n");
+  return `${AD_INJECTION_MARKER}
+${paragraphs}`;
+}
+function buildJsonLdBlock(ads) {
+  const entries = ads.map((ad) => ({
+    "@context": "https://schema.org",
+    "@type": "WPAdBlock",
+    sponsor: {
+      "@type": "Organization",
+      name: ad.advertiser,
+      url: ad.url
+    },
+    description: ad.text
+  }));
+  const ld = entries.length === 1 ? entries[0] : entries;
+  return `<script type="application/ld+json">${JSON.stringify(ld)}</script>`;
+}
+function escapeHtml(s) {
+  return s.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;").replace(/'/g, "&#39;");
+}
+function escapeAttr(s) {
+  return s.replace(/"/g, "&quot;").replace(/'/g, "&#39;");
+}
+
 // src/middleware/express.ts
 var instances2 = /* @__PURE__ */ new Map();
 function getInstance2(config) {