@lincoln504/pi-research 1.5.3 → 1.6.0

package/src/infrastructure/browser/thread-worker.mjs

@@ -1516,11 +1516,25 @@ function isPoolShutdownError(error) {
   // same transient swap the "being reset" string covers, not a hard fault.
   err.message.includes("Cannot execute a task on not started pool"));
 }
-var DEFAULT_BREAKER_CONFIG, browserCircuitBreaker;
+function isBenignScrapeFailure(error) {
+  const msg = error instanceof Error ? error.message : String(error);
+  return msg.startsWith("Fetch blocked:") || // bot-protection interstitial (Cloudflare, DDoS-Guard, …)
+  msg.startsWith("Fetch returned stub:") || // nav-only / near-empty page
+  msg.startsWith("Could not extract content from PDF") || // malformed/encrypted/scanned-only PDF
+  msg.startsWith("PDF too large") || // exceeds the deliberate 100MB policy cap
+  msg.startsWith("Unsupported download:") || // attachment-served non-PDF (zip/xlsx/…) — nothing to extract
+  BENIGN_NAVIGATION_FAILURE.test(msg) || // remote TLS/DNS/timeout — the URL could not be loaded
+  msg.startsWith("HTML response too large") || // fetch-layer 25MB policy cap — same class as the PDF cap
+  msg.startsWith("Browser HTML too large") || // browser-layer 25MB policy cap
+  /^HTTP \d{3}\b/.test(msg) || // remote 4xx/5xx status
+  isCloudflareBlockError(error) || isTaskTimeoutError(error) || isPoolShutdownError(error);
+}
+var BENIGN_NAVIGATION_FAILURE, DEFAULT_BREAKER_CONFIG, browserCircuitBreaker;
 var init_browser_error_utils = __esm({
   "src/infrastructure/browser/browser-error-utils.ts"() {
     "use strict";
     init_circuit_breaker();
+    BENIGN_NAVIGATION_FAILURE = /page\.goto: (?:Timeout \d+ *ms exceeded|net::|NS_ERROR_|SEC_ERROR_|MOZILLA_PKIX_ERROR_)/;
     DEFAULT_BREAKER_CONFIG = {
       // Raised from 3→8: parallel search bursts of 20 queries frequently produce
       // simultaneous timeouts on slow/CF-protected sites. A threshold of 3 caused
@@ -1565,17 +1579,6 @@ import * as dnsCb from "node:dns";
 import * as net from "node:net";
 import crypto from "node:crypto";
 import { NodeHtmlMarkdown } from "node-html-markdown";
-function isBenignScrapeFailure(error) {
-  const msg = error instanceof Error ? error.message : String(error);
-  return msg.startsWith("Fetch blocked:") || // bot-protection interstitial (Cloudflare, DDoS-Guard, …)
-  msg.startsWith("Fetch returned stub:") || // nav-only / near-empty page
-  msg.startsWith("Could not extract content from PDF") || // malformed/encrypted/scanned-only PDF
-  msg.startsWith("PDF too large") || // exceeds the deliberate 100MB policy cap
-  msg.startsWith("HTML response too large") || // fetch-layer 25MB policy cap — same class as the PDF cap
-  msg.startsWith("Browser HTML too large") || // browser-layer 25MB policy cap
-  /^HTTP \d{3}\b/.test(msg) || // remote 4xx/5xx status
-  isCloudflareBlockError(error) || isTaskTimeoutError(error) || isPoolShutdownError(error);
-}
 function getRandomUserAgent() {
   try {
     const index = crypto.randomInt(0, USER_AGENTS.length);
@@ -2014,7 +2017,7 @@ import { platform as platform2 } from "node:os";
 
 // src/infrastructure/browser/thread-worker-messaging.ts
 init_log_utils();
-import { appendFileSync as appendFileSync2 } from "node:fs";
+import { appendFileSync as appendFileSync2, promises as fsp } from "node:fs";
 
 // src/utils/safe-unref.ts
 function safeUnref(timer) {
@@ -2286,6 +2289,66 @@ function seedSubresourceVerdict(validatedUrl) {
   } catch {
   }
 }
+function isEvictedBodyError(err) {
+  const msg = err instanceof Error ? err.message : String(err);
+  return /was evicted|Response body is unavailable/i.test(msg);
+}
+var PDF_REFETCH_TIMEOUT_MS = 9e4;
+async function refetchPdfBody(page, url, validateForWorker, signal) {
+  await validateForWorker(url);
+  if (signal?.aborted) throw new Error("Aborted");
+  const requestPromise = page.context().request.get(url, {
+    maxRedirects: 0,
+    timeout: PDF_REFETCH_TIMEOUT_MS
+  });
+  requestPromise.catch(() => {
+  });
+  const response = await (signal ? Promise.race([
+    requestPromise,
+    new Promise((_, reject) => {
+      if (signal.aborted) return reject(new Error("Aborted"));
+      signal.addEventListener("abort", () => reject(new Error("Aborted")), { once: true });
+    })
+  ]) : requestPromise);
+  const status = Number(response.status?.() ?? 0);
+  if (!response.ok?.()) throw new Error(`HTTP ${status}`);
+  const declared = parseInt(String((await response.headersArray?.() ?? []).find?.((h) => String(h.name).toLowerCase() === "content-length")?.value ?? ""), 10);
+  if (Number.isFinite(declared) && declared > MAX_PDF_SIZE) {
+    throw new Error(`PDF too large (${Math.round(declared / 1024 / 1024)}MB, max 100MB)`);
+  }
+  const buffer = await response.body();
+  if (buffer.byteLength > MAX_PDF_SIZE) {
+    throw new Error(`PDF too large (${Math.round(buffer.byteLength / 1024 / 1024)}MB, max 100MB)`);
+  }
+  logToDebugFile2("DEBUG", `[Worker-${workerId2}] Re-fetched evicted PDF body (${buffer.byteLength} bytes) for: ${url}`);
+  return buffer;
+}
+async function readPdfDownload(download, url, page, pendingAddrChecks, getPoisonedError) {
+  const failure = await download.failure?.().catch(() => null);
+  if (failure) throw new Error(`Fetch blocked: download failed (${failure})`);
+  const suggested = String(download.suggestedFilename?.() ?? "");
+  const downloadUrl = String(download.url?.() ?? url);
+  const looksLikePdf = suggested.toLowerCase().endsWith(".pdf") || downloadUrl.toLowerCase().split("?")[0].endsWith(".pdf");
+  if (!looksLikePdf) {
+    throw new Error(`Unsupported download: ${suggested || downloadUrl} is not a PDF`);
+  }
+  const filePath = await download.path();
+  if (!filePath) throw new Error("Fetch blocked: download produced no readable file");
+  const { size } = await fsp.stat(filePath);
+  if (size > MAX_PDF_SIZE) {
+    const sizeMB = Math.round(size / 1024 / 1024);
+    throw new Error(`PDF too large (${sizeMB}MB, max 100MB)`);
+  }
+  const buffer = await fsp.readFile(filePath);
+  await Promise.allSettled(pendingAddrChecks);
+  const poisoned = getPoisonedError();
+  if (poisoned) throw poisoned;
+  await download.delete?.().catch(() => {
+  });
+  await page.close();
+  logToDebugFile2("DEBUG", `[Worker-${workerId2}] Recovered attachment-served PDF (${buffer.byteLength} bytes) for: ${url}`);
+  return { contentType: "application/pdf", bufferB64: buffer.toString("base64"), jitter: 0 };
+}
 async function executeScrapeTask(_context, url, signal) {
   const page = await createPageSafe(_context);
   const SCRAPE_TIMEOUT = parseTimeoutMs(process.env["PI_RESEARCH_SCRAPE_TIMEOUT_MS"], 15e3);
@@ -2302,6 +2365,7 @@ async function executeScrapeTask(_context, url, signal) {
   let poisonedError = null;
   const pendingAddrChecks = [];
   let oversizedPdfError = null;
+  let pendingDownload = null;
   try {
     logToDebugFile2("DEBUG", `[Worker-${workerId2}] Starting scrape for: ${url}`);
     const { validateUrlForSSRF: validateForWorker, validateUrlForSSRFSync: validateForWorkerSync } = await Promise.resolve().then(() => (init_scraper_utils(), scraper_utils_exports));
@@ -2368,7 +2432,25 @@ async function executeScrapeTask(_context, url, signal) {
       } catch {
       }
     });
-    const response = await page.goto(url, { waitUntil: "domcontentloaded" });
+    page.on("download", (d) => {
+      pendingDownload ??= d;
+    });
+    let response;
+    try {
+      response = await page.goto(url, { waitUntil: "domcontentloaded" });
+    } catch (navErr) {
+      const navMsg = navErr instanceof Error ? navErr.message : String(navErr);
+      const isDownloadNav = /download is starting/i.test(navMsg);
+      if (!isDownloadNav) throw navErr;
+      if (!pendingDownload) {
+        await Promise.race([
+          page.waitForEvent("download", { timeout: 5e3 }).catch(() => null),
+          new Promise((r) => setTimeout(r, 5e3))
+        ]);
+      }
+      if (!pendingDownload) throw navErr;
+      return await readPdfDownload(pendingDownload, url, page, pendingAddrChecks, () => poisonedError);
+    }
     const serverAddr = await response?.serverAddr().catch(() => null);
     if (serverAddr?.ipAddress) {
       const ip = serverAddr.ipAddress;
@@ -2386,7 +2468,13 @@ async function executeScrapeTask(_context, url, signal) {
         throw new Error(`PDF too large (${sizeMB}MB, max 100MB)`);
       }
       if (oversizedPdfError) throw oversizedPdfError;
-      const buffer = await response.body();
+      let buffer;
+      try {
+        buffer = await response.body();
+      } catch (bodyErr) {
+        if (!isEvictedBodyError(bodyErr)) throw bodyErr;
+        buffer = await refetchPdfBody(page, finalUrl, validateForWorker, signal);
+      }
       if (buffer.byteLength > MAX_PDF_SIZE) {
         const sizeMB = Math.round(buffer.byteLength / 1024 / 1024);
         throw new Error(`PDF too large (${sizeMB}MB, max 100MB)`);
@@ -2749,7 +2837,7 @@ function logToDebugFile3(level, ...args) {
         return String(arg);
       }).join(" "))
     };
-    import("node:fs/promises").then((fsp) => fsp.appendFile(logFile, `${JSON.stringify(entry)}
+    import("node:fs/promises").then((fsp2) => fsp2.appendFile(logFile, `${JSON.stringify(entry)}
 `)).catch(() => {
     });
   } catch {
@@ -2888,7 +2976,7 @@ async function initBrowser() {
   return initPromise;
 }
 async function acquireTaskContext() {
-  const contextPromise = browser.newContext();
+  const contextPromise = browser.newContext({ acceptDownloads: true });
   contextPromise.catch((err) => logToDebugFile3("DEBUG", `[ThreadWorker] Background browser context rejection: ${err.message}`));
   let contextTimeoutId;
   let contextRaceLost = false;

package/src/knowledge/store-operations.ts

@@ -13,6 +13,13 @@ import { logger } from '../logger.ts';
 import { metrics } from '../utils/metrics.ts';
 import { StoreDocument } from '../core/interfaces/knowledge-interfaces.ts';
 
+/**
+ * What a single add attempt actually did. Returned rather than logged/counted here
+ * so the caller — which owns the retry loop — records exactly one outcome per
+ * logical addDocuments call.
+ */
+export type AddDocumentsOutcome = 'written' | 'empty' | 'ignored_closing';
+
 /**
  * Guard against a broken embedder silently writing corrupt vectors: a
  * wrong-count, empty, ragged, or NaN/Inf-poisoned result would otherwise land
@@ -69,12 +76,11 @@ export async function addDocumentsToStore(
   isClosing: () => boolean,
   workspace: string,
   isGlobal: boolean
-): Promise<void> {
-  if (docs.length === 0) return;
+): Promise<AddDocumentsOutcome> {
+  if (docs.length === 0) return 'empty';
   if (isClosing()) {
     logger.warn('[store] Ignoring addDocuments during close');
-    metrics.increment('knowledge_store_add_documents_total', 1, { status: 'ignored_closing' });
-    return;
+    return 'ignored_closing';
   }
 
   const startTime = Date.now();
@@ -98,14 +104,19 @@ export async function addDocumentsToStore(
     await table.add(data);
     const duration = Date.now() - startTime;
     metrics.observe('knowledge_store_add_documents_duration_ms', duration);
-    metrics.increment('knowledge_store_add_documents_total', 1, { status: 'success' });
     metrics.increment('knowledge_store_chunks_added_total', docs.length);
     logger.log(`[store] Added ${docs.length} chunk(s) for ${docs[0]?.url} [workspace=${workspace}, global=${isGlobal}]`);
+    return 'written';
   } catch (err) {
     const duration = Date.now() - startTime;
     metrics.observe('knowledge_store_add_documents_duration_ms', duration, { status: 'error' });
-    metrics.increment('knowledge_store_add_documents_total', 1, { status: 'error' });
-    logger.error('[store] Failed to add documents:', err);
+    // Attempt-level failure only. The caller retries embedder-unreachable and Lance
+    // commit-conflict failures and usually succeeds, so logging ERROR and counting a
+    // terminal outcome here reported lost writes that were not lost, and double-counted
+    // the batch in knowledge_store_add_documents_total (once error, once success).
+    // Outcome accounting belongs to KnowledgeStore.addDocuments, which is the only
+    // layer that knows whether the write was ultimately abandoned.
+    logger.debug(`[store] add attempt failed after ${duration}ms: ${err instanceof Error ? err.message : String(err)}`);
     throw err;
   }
 }

package/src/knowledge/store.ts

@@ -23,6 +23,7 @@ import { MigrationStrategy, MigrationResult } from './migration.ts';
 import { metrics } from '../utils/metrics.ts';
 import { createStoreTable, CURRENT_SCHEMA_VERSION } from './store-schema.ts';
 import { addDocumentsToStore, assertValidEmbeddingVectors, searchStore, findDocumentsByUrl, findRelevantUrls } from './store-operations.ts';
+import type { AddDocumentsOutcome } from './store-operations.ts';
 import { isEmbedderUnreachable } from './embedder-utils.ts';
 import { ServiceLifecycle } from '../core/service-registry.ts';
 import { ServiceNames } from '../core/service-interfaces.ts';
@@ -713,13 +714,14 @@ export class KnowledgeStore implements IKnowledgeStore {
       // Now check isClosing after registering
       if (this.isClosing) {
         logger.warn(`[store] Skipping addDocuments — store is closing`);
+        metrics.increment('knowledge_store_add_documents_total', 1, { status: 'ignored_closing' });
         return;
       }
 
       let table = await this.getFreshTable();
       const workspace = this.getWorkspace();
 
-      await this.withEmbedderReconnect(async (embedder) => {
+      const outcome = await this.withEmbedderReconnect<AddDocumentsOutcome>(async (embedder) => {
         let retryCount = 0;
         const MAX_RETRIES = 5;
         const BASE_DELAY = 100;
@@ -727,7 +729,7 @@ export class KnowledgeStore implements IKnowledgeStore {
         while (retryCount <= MAX_RETRIES) {
           try {
             const isGlobal = this.options.knowledgeMode === 'global';
-          await addDocumentsToStore(
+            return await addDocumentsToStore(
               table, 
               docs, 
               embedder, 
@@ -735,7 +737,6 @@ export class KnowledgeStore implements IKnowledgeStore {
               workspace,
               isGlobal
             );
-            return;
           } catch (err) {
             // Detect commit conflicts typical of cross-process write contention
             if (isLanceCommitConflict(err)) {
@@ -758,7 +759,24 @@ export class KnowledgeStore implements IKnowledgeStore {
             throw err;
           }
         }
+        // Unreachable: the loop only exits via return or throw. Present so the
+        // outcome type stays non-optional for the accounting below.
+        throw new Error('[store] addDocuments retry loop exited without result');
       });
+
+      // Exactly one outcome per logical addDocuments call. Attempts that failed and
+      // were then recovered by the reconnect/conflict retries are not counted here
+      // and are not logged at ERROR — only a write that is actually abandoned is.
+      // An empty batch is a no-op and is not counted at all, as before.
+      if (outcome !== 'empty') {
+        metrics.increment('knowledge_store_add_documents_total', 1, {
+          status: outcome === 'ignored_closing' ? 'ignored_closing' : 'success',
+        });
+      }
+    } catch (err) {
+      metrics.increment('knowledge_store_add_documents_total', 1, { status: 'error' });
+      logger.error('[store] Failed to add documents:', err);
+      throw err;
     } finally {
       this.activeWrites.delete(writePromise);
       writeResolve!();

package/src/web-research/scraper-utils.ts

@@ -19,45 +19,13 @@ import {
   INTERNAL_NETWORK_PATTERNS,
   type NativeHtmlToMarkdownModule,
 } from './scraper-types.ts';
-import { isCloudflareBlockError, isTaskTimeoutError, isPoolShutdownError } from '../infrastructure/browser/browser-error-utils.ts';
 
 /**
- * True when a scrape failure is an EXPECTED per-URL outcome rather than an engine
- * fault. Scraping the open web routinely hits bot-protection interstitials, thin
- * "stub" pages, HTTP 4xx/5xx, slow/unresponsive remotes, and (on quit) pool-drain
- * races. None of these is a pi-research bug: the URL is already accounted for as
- * "not scraped" (scrape_results_total{outcome=total_failure}) and surfaced to the
- * user that way, and no operator action follows. Callers use this to log such
- * failures at debug instead of error — keeping the ERROR log (and, via
- * logger.error's error-tracker re-track, the diagnostic error count) focused on
- * genuine faults. The messages matched here are produced by validateContent
- * (`Fetch blocked:` / `Fetch returned stub:`), scrapeWithFetch (`HTTP <code>`)
- * and extractPdfToMarkdown (`Could not extract content from PDF`, `PDF too
- * large`) — a malformed/encrypted/scanned-image-only PDF, or one that simply
- * exceeds the deliberate 100MB policy cap, is routine on the open web and
- * just as non-actionable as a stub page or a 404; the 25MB HTML policy caps
- * (`HTML response too large` from the fetch layer, `Browser HTML too large`
- * from the browser layer) are the same deliberate-cap class; the remaining
- * cases delegate to the shared browser-layer predicates. Deliberately does NOT
- * match 'PDF extraction unavailable' (a pdf-oxide-wasm native-module load
- * failure) — that is an infrastructure fault affecting every URL, not a
- * routine per-PDF outcome, and must still surface at ERROR.
+ * Re-exported from the browser error utilities, where it lives beside the three
+ * predicates it delegates to. Kept exported here because this is where every scrape
+ * caller already imports its error helpers from.
  */
-export function isBenignScrapeFailure(error: unknown): boolean {
-  const msg = error instanceof Error ? error.message : String(error);
-  return (
-    msg.startsWith('Fetch blocked:') ||                  // bot-protection interstitial (Cloudflare, DDoS-Guard, …)
-    msg.startsWith('Fetch returned stub:') ||             // nav-only / near-empty page
-    msg.startsWith('Could not extract content from PDF') || // malformed/encrypted/scanned-only PDF
-    msg.startsWith('PDF too large') ||                    // exceeds the deliberate 100MB policy cap
-    msg.startsWith('HTML response too large') ||          // fetch-layer 25MB policy cap — same class as the PDF cap
-    msg.startsWith('Browser HTML too large') ||           // browser-layer 25MB policy cap
-    /^HTTP \d{3}\b/.test(msg) ||                          // remote 4xx/5xx status
-    isCloudflareBlockError(error) ||
-    isTaskTimeoutError(error) ||
-    isPoolShutdownError(error)
-  );
-}
+export { isBenignScrapeFailure } from '../infrastructure/browser/browser-error-utils.ts';
 
 /**
  * Get a random user agent

package/src/youtube/potoken.ts

@@ -6,8 +6,8 @@
  * HTTP 200 with an EMPTY body (a silent failure). This module mints tokens with
  * `bgutils-js`, running the BotGuard VM under a `jsdom` DOM.
  *
- * Hard-won implementation details (validated live against youtubei.js@17 /
- * bgutils-js@3 / jsdom@29):
+ * Hard-won implementation details (re-validated live against youtubei.js@18 /
+ * bgutils-js@4 / jsdom@29):
  *
  *  1. Cross-realm typed arrays. The BotGuard VM must run in the SAME JS realm as
  *     bgutils' `base64ToU8` (the Node realm). So the interpreter is eval'd via
@@ -171,9 +171,17 @@ async function doMint(
   const started = Date.now();
 
   // Lazy-load heavy deps — never imported at module load (keeps startup native-free).
-  const bgMod = (await import('bgutils-js')) as unknown as BgModule;
-  const BG = bgMod.BG ?? bgMod.default;
-  const { buildURL, getHeaders } = bgMod;
+  // bgutils-js 4 dropped the barrel entry (package.json "exports" has no "."), so each
+  // piece comes from its own subpath: the challenge fetcher and VM client from
+  // /botguard, the minter from /webpo, the attestation URL/header helpers from /utils.
+  const [bgGuard, bgWebPo, bgUtils] = (await Promise.all([
+    import('bgutils-js/botguard'),
+    import('bgutils-js/webpo'),
+    import('bgutils-js/utils'),
+  ])) as unknown as [BgGuardModule, BgWebPoModule, BgUtilsModule];
+  const { getChallenge, BotGuardClient } = bgGuard;
+  const { WebPoMinter } = bgWebPo;
+  const { buildURL, getHeaders } = bgUtils;
   const { JSDOM } = await import('jsdom');
 
   const dom = new JSDOM('<!DOCTYPE html><html lang="en"><body></body></html>', {
@@ -211,18 +219,18 @@ async function doMint(
 
   bridge();
   try {
-    const bgConfig: BgConfig = {
-      fetch: (input: unknown, init?: unknown) =>
+    // bgutils-js 4 replaced BG.Challenge.create(config) with a free getChallenge()
+    // whose config is narrower: no globalObj and no identifier (the content binding is
+    // supplied per-mint, which is what this module already did), and `fetch` renamed
+    // to `fetchFunction`.
+    const challenge = await getChallenge({
+      requestKey,
+      fetchFunction: ((input: unknown, init?: unknown) =>
         fetchImpl(input as Parameters<typeof fetch>[0], {
           ...(init as RequestInit | undefined),
           signal: (init as RequestInit | undefined)?.signal ?? opts.signal,
-        }),
-      globalObj: globalThis,
-      identifier: identifiers[0]!,
-      requestKey,
-    };
-
-    const challenge = await BG.Challenge.create(bgConfig);
+        })) as unknown as typeof fetch,
+    });
     if (!challenge) throw new Error('BotGuard challenge unavailable');
 
     const interpreter = challenge.interpreterJavascript?.privateDoNotAccessOrElseSafeScriptWrappedValue;
@@ -233,10 +241,16 @@ async function doMint(
     // PoToken generation; trust rests on that TLS connection.
     new Function(interpreter)();
 
-    const botguard = await BG.BotGuardClient.create({
+    if (!challenge.program || !challenge.globalName) {
+      throw new Error('BotGuard challenge missing program/globalName');
+    }
+    // `globalObj` was renamed `globalObject` in bgutils-js 4. Passing the old name
+    // leaves the VM without a global to attach to and the minter factory silently
+    // returns a non-function, so this rename is load-bearing, not cosmetic.
+    const botguard = await BotGuardClient.create({
       program: challenge.program,
       globalName: challenge.globalName,
-      globalObj: globalThis,
+      globalObject: globalThis,
     });
 
     const webPoSignalOutput: unknown[] = [];
@@ -260,7 +274,7 @@ async function doMint(
       throw new Error('Integrity token rejected by attestation server (likely a non-residential IP)');
     }
 
-    const minter = await BG.WebPoMinter.create({ integrityToken }, webPoSignalOutput);
+    const minter = await WebPoMinter.create({ integrityToken }, webPoSignalOutput);
 
     const tokens = new Map<string, string>();
     for (const id of identifiers) {
@@ -294,26 +308,27 @@ async function doMint(
 // Minimal structural types for the loosely-typed bgutils-js surface we touch.
 // ---------------------------------------------------------------------------
 
-interface BgConfig {
-  fetch: (input: unknown, init?: unknown) => Promise<Response>;
-  globalObj: unknown;
-  identifier: string;
-  requestKey: string;
+interface BgChallenge {
+  interpreterJavascript?: { privateDoNotAccessOrElseSafeScriptWrappedValue?: string };
+  program?: string;
+  globalName?: string;
 }
 
-interface BgNamespace {
-  Challenge: {
-    create(config: BgConfig): Promise<{
-      interpreterJavascript?: { privateDoNotAccessOrElseSafeScriptWrappedValue?: string };
-      program: string;
-      globalName: string;
-    } | undefined>;
-  };
+interface BgGuardModule {
+  getChallenge(config: {
+    requestKey: string;
+    fetchFunction: typeof fetch;
+    interpreterHash?: string;
+    useYouTubeAPI?: boolean;
+  }): Promise<BgChallenge | undefined>;
   BotGuardClient: {
-    create(opts: { program: string; globalName: string; globalObj: unknown }): Promise<{
+    create(opts: { program: string; globalName: string; globalObject: unknown }): Promise<{
       snapshot(args: { webPoSignalOutput: unknown[] }): Promise<string>;
     }>;
   };
+}
+
+interface BgWebPoModule {
   WebPoMinter: {
     create(
       integrity: { integrityToken: string },
@@ -322,9 +337,7 @@ interface BgNamespace {
   };
 }
 
-interface BgModule {
-  BG?: BgNamespace;
-  default: BgNamespace;
+interface BgUtilsModule {
   buildURL(endpoint: string, useYouTubeAPI?: boolean): string;
   getHeaders(): Record<string, string>;
 }