extract-webpage 1.2.116 → 1.2.117

package/README.md

@@ -2,11 +2,8 @@
 <br /> 
     <a href="https://www.npmjs.com/package/extract-webpage"><img src="https://img.shields.io/npm/dm/extract-webpage.svg" alt="NPM Monthly Downloads"></a>
     <a href="https://www.npmjs.com/package/extract-webpage"><img src="https://img.shields.io/npm/v/extract-webpage.svg" alt="npm version"></a>
-    <a href="https://bundlephobia.com/package/extract-webpage" target="_blank" rel="noopener noreferrer">
-  <img
-    src="https://img.shields.io/bundlephobia/minzip/extract-webpage?style=flat&label=size"
-    alt="npm bundle size "
-  />
+    <a href="https://packagephobia.com/result?p=extract-webpage" target="_blank" rel="noopener noreferrer">
+  <img src="https://packagephobia.com/badge?p=extract-webpage" alt="install size" />
 </a>
     <a href="https://discord.gg/SJdBqBz3tV">
         <img src="https://img.shields.io/discord/1110227955554209923.svg?label=Chat&logo=Discord&colorB=7289da&style=flat"

package/dist/url-to-content/cloudflare-scraper-client.d.ts

@@ -0,0 +1,112 @@
+/**
+ * @fileoverview Client for the Cloudflare Puppeteer-based scraper service.
+ * Renders JavaScript-heavy pages and bypasses bot detection.
+ *
+ * This wraps the scraper-cloudflare package deployed as a Cloudflare Worker
+ * with Browser Rendering (Puppeteer) support.
+ */
+export interface ScraperOptions {
+    /** URL to render */
+    url: string;
+    /** API key for authentication (optional if SCRAPER_API_KEY not set) */
+    apiKey?: string;
+    /** Additional wait time after page load (ms) */
+    wait?: number;
+    /** Block image loading for faster rendering */
+    blockImages?: boolean;
+    /** Session ID for cookie persistence */
+    sessionId?: string;
+    /** Navigation timeout (ms) */
+    timeout?: number;
+    /** Puppeteer waitUntil condition */
+    waitUntil?: 'domcontentloaded' | 'load' | 'networkidle0' | 'networkidle2';
+    /** Response format */
+    format?: 'html' | 'json';
+    /** Custom headers */
+    headers?: Record<string, string>;
+    /** Proxy configuration */
+    proxyUrl?: string;
+    proxyUser?: string;
+    proxyPass?: string;
+    /** Bypass Cloudflare challenges and CAPTCHAs */
+    bypassCaptcha?: boolean;
+    /** Challenge detection pattern */
+    challengeMatch?: string;
+    /** Max retry attempts for challenges */
+    maxRetries?: number;
+    /** 2Captcha API key for solving */
+    twoCaptchaKey?: string;
+    /** Abort signal to bound the request (e.g. an 8s deadline). */
+    signal?: AbortSignal;
+}
+export interface ScraperJsonResponse {
+    html: string;
+    url: string;
+    title: string;
+    cookies: Array<{
+        name: string;
+        value: string;
+        domain: string;
+        path: string;
+        expires?: number;
+        httpOnly?: boolean;
+        secure?: boolean;
+        sameSite?: 'Strict' | 'Lax' | 'None';
+    }>;
+    challengeBypassed: boolean;
+    retryCount: number;
+    loadTime: number;
+}
+export interface ScraperConfig {
+    /** Base URL of the scraper service */
+    baseURL: string;
+    /** Global API key */
+    apiKey?: string;
+}
+/**
+ * Renders a URL using the Cloudflare Puppeteer scraper service.
+ * Supports JavaScript rendering, bot detection bypass, and session management.
+ *
+ * @param options - Scraping configuration
+ * @param config - Service configuration (base URL and API key)
+ * @returns Rendered HTML or structured JSON response
+ *
+ * @example
+ * ```ts
+ * // Basic usage
+ * const html = await renderWithCloudflare({ url: 'https://example.com' });
+ *
+ * // With challenge bypass
+ * const result = await renderWithCloudflare({
+ *   url: 'https://protected-site.com',
+ *   bypassCaptcha: true,
+ *   format: 'json'
+ * });
+ *
+ * // With session management
+ * const html = await renderWithCloudflare({
+ *   url: 'https://site-requiring-login.com',
+ *   sessionId: 'user-123',
+ *   blockImages: true
+ * });
+ * ```
+ */
+export declare function renderWithCloudflare(options: ScraperOptions, config?: Partial<ScraperConfig>): Promise<string | ScraperJsonResponse>;
+/**
+ * Convenience function to render a URL and return just the HTML content.
+ *
+ * @param url - URL to render
+ * @param options - Additional scraping options
+ * @param config - Service configuration
+ * @returns Rendered HTML string
+ */
+export declare function renderUrlToHtml(url: string, options?: Omit<ScraperOptions, 'url'>, config?: Partial<ScraperConfig>): Promise<string>;
+/**
+ * Renders a URL and returns full metadata including cookies, load time, etc.
+ *
+ * @param url - URL to render
+ * @param options - Additional scraping options
+ * @param config - Service configuration
+ * @returns Structured response with HTML and metadata
+ */
+export declare function renderUrlWithMetadata(url: string, options?: Omit<ScraperOptions, 'url' | 'format'>, config?: Partial<ScraperConfig>): Promise<ScraperJsonResponse>;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "extract-webpage",
-  "version": "1.2.116",
+  "version": "1.2.117",
   "module": "./dist/extract-webpage.es.js",
   "description": "Search, extract, cite, and outline the web for a topic with AI Research Agent.",
   "author": "vtempest <grokthiscontact@gmail.com>",
@@ -72,11 +72,11 @@
   "dependencies": {
     "@huggingface/transformers": "^3.8.1",
     "ai": "^5.0.0",
-    "chat-agent-toolkit": "^1.2.116",
+    "chat-agent-toolkit": "^1.2.117",
     "chrono-node": "^2.9.0",
     "drizzle-orm": "^0.45.1",
-    "extract-pdf": "^0.1.103",
-    "extract-youtube": "^1.0.99",
+    "extract-pdf": "^0.1.104",
+    "extract-youtube": "^1.0.102",
     "html-entities": "^2.6.0",
     "js-yaml": "^4.1.1",
     "jsdom": "^28.1.0",

package/src/url-to-content/cloudflare-scraper-client.ts

@@ -0,0 +1,205 @@
+/**
+ * @fileoverview Client for the Cloudflare Puppeteer-based scraper service.
+ * Renders JavaScript-heavy pages and bypasses bot detection.
+ *
+ * This wraps the scraper-cloudflare package deployed as a Cloudflare Worker
+ * with Browser Rendering (Puppeteer) support.
+ */
+
+export interface ScraperOptions {
+  /** URL to render */
+  url: string;
+  /** API key for authentication (optional if SCRAPER_API_KEY not set) */
+  apiKey?: string;
+  /** Additional wait time after page load (ms) */
+  wait?: number;
+  /** Block image loading for faster rendering */
+  blockImages?: boolean;
+  /** Session ID for cookie persistence */
+  sessionId?: string;
+  /** Navigation timeout (ms) */
+  timeout?: number;
+  /** Puppeteer waitUntil condition */
+  waitUntil?: 'domcontentloaded' | 'load' | 'networkidle0' | 'networkidle2';
+  /** Response format */
+  format?: 'html' | 'json';
+  /** Custom headers */
+  headers?: Record<string, string>;
+  /** Proxy configuration */
+  proxyUrl?: string;
+  proxyUser?: string;
+  proxyPass?: string;
+  /** Bypass Cloudflare challenges and CAPTCHAs */
+  bypassCaptcha?: boolean;
+  /** Challenge detection pattern */
+  challengeMatch?: string;
+  /** Max retry attempts for challenges */
+  maxRetries?: number;
+  /** 2Captcha API key for solving */
+  twoCaptchaKey?: string;
+  /** Abort signal to bound the request (e.g. an 8s deadline). */
+  signal?: AbortSignal;
+}
+
+export interface ScraperJsonResponse {
+  html: string;
+  url: string;
+  title: string;
+  cookies: Array<{
+    name: string;
+    value: string;
+    domain: string;
+    path: string;
+    expires?: number;
+    httpOnly?: boolean;
+    secure?: boolean;
+    sameSite?: 'Strict' | 'Lax' | 'None';
+  }>;
+  challengeBypassed: boolean;
+  retryCount: number;
+  loadTime: number;
+}
+
+export interface ScraperConfig {
+  /** Base URL of the scraper service */
+  baseURL: string;
+  /** Global API key */
+  apiKey?: string;
+}
+
+const DEFAULT_CONFIG: ScraperConfig = {
+  baseURL: typeof process !== 'undefined' && process?.env?.SCRAPER_URL
+    ? process.env.SCRAPER_URL
+    : 'https://proxy.qwksearch.com',
+  apiKey: typeof process !== 'undefined' && process?.env?.SCRAPER_API_KEY
+    ? process.env.SCRAPER_API_KEY
+    : undefined,
+};
+
+/**
+ * Renders a URL using the Cloudflare Puppeteer scraper service.
+ * Supports JavaScript rendering, bot detection bypass, and session management.
+ *
+ * @param options - Scraping configuration
+ * @param config - Service configuration (base URL and API key)
+ * @returns Rendered HTML or structured JSON response
+ *
+ * @example
+ * ```ts
+ * // Basic usage
+ * const html = await renderWithCloudflare({ url: 'https://example.com' });
+ *
+ * // With challenge bypass
+ * const result = await renderWithCloudflare({
+ *   url: 'https://protected-site.com',
+ *   bypassCaptcha: true,
+ *   format: 'json'
+ * });
+ *
+ * // With session management
+ * const html = await renderWithCloudflare({
+ *   url: 'https://site-requiring-login.com',
+ *   sessionId: 'user-123',
+ *   blockImages: true
+ * });
+ * ```
+ */
+export async function renderWithCloudflare(
+  options: ScraperOptions,
+  config: Partial<ScraperConfig> = {}
+): Promise<string | ScraperJsonResponse> {
+  const mergedConfig = { ...DEFAULT_CONFIG, ...config };
+  const apiKey = options.apiKey || mergedConfig.apiKey;
+
+  // The deployed scraper worker (proxy.qwksearch.com) accepts GET requests
+  // with query-string parameters on `/` and `/api/render`.
+  const url = new URL('/api/render', mergedConfig.baseURL);
+
+  const params: Record<string, string | number | boolean | undefined> = {
+    url: options.url,
+    wait: options.wait ?? 0,
+    blockImages: options.blockImages ?? false,
+    sessionId: options.sessionId ?? 'default',
+    timeout: options.timeout ?? 30000,
+    waitUntil: options.waitUntil ?? 'networkidle2',
+    format: options.format ?? 'html',
+    proxyUrl: options.proxyUrl,
+    proxyUser: options.proxyUser,
+    proxyPass: options.proxyPass,
+    bypassCaptcha: options.bypassCaptcha ?? true,
+    challengeMatch: options.challengeMatch,
+    maxRetries: options.maxRetries ?? 10,
+    twoCaptchaKey: options.twoCaptchaKey,
+  };
+
+  for (const [key, value] of Object.entries(params)) {
+    if (value !== undefined) {
+      url.searchParams.set(key, String(value));
+    }
+  }
+
+  const headers: Record<string, string> = { ...(options.headers ?? {}) };
+
+  if (apiKey) {
+    headers['Authorization'] = `Bearer ${apiKey}`;
+  }
+
+  const response = await fetch(url.toString(), {
+    method: 'GET',
+    headers,
+    signal: options.signal,
+  });
+
+  if (!response.ok) {
+    const errorText = await response.text();
+    throw new Error(
+      `Scraper request failed (${response.status}): ${errorText}`
+    );
+  }
+
+  if (options.format === 'json') {
+    return (await response.json()) as ScraperJsonResponse;
+  }
+
+  return await response.text();
+}
+
+/**
+ * Convenience function to render a URL and return just the HTML content.
+ *
+ * @param url - URL to render
+ * @param options - Additional scraping options
+ * @param config - Service configuration
+ * @returns Rendered HTML string
+ */
+export async function renderUrlToHtml(
+  url: string,
+  options: Omit<ScraperOptions, 'url'> = {},
+  config: Partial<ScraperConfig> = {}
+): Promise<string> {
+  const result = await renderWithCloudflare(
+    { ...options, url, format: 'html' },
+    config
+  );
+  return typeof result === 'string' ? result : result.html;
+}
+
+/**
+ * Renders a URL and returns full metadata including cookies, load time, etc.
+ *
+ * @param url - URL to render
+ * @param options - Additional scraping options
+ * @param config - Service configuration
+ * @returns Structured response with HTML and metadata
+ */
+export async function renderUrlWithMetadata(
+  url: string,
+  options: Omit<ScraperOptions, 'url' | 'format'> = {},
+  config: Partial<ScraperConfig> = {}
+): Promise<ScraperJsonResponse> {
+  const result = await renderWithCloudflare(
+    { ...options, url, format: 'json' },
+    config
+  );
+  return result as ScraperJsonResponse;
+}