@graphext/cuery 0.9.5 → 0.10.1

package/esm/browser.d.ts

@@ -2,7 +2,7 @@
  * Browser-safe exports for @graphext/cuery
  *
  * This module only exports types and pure functions that can safely run in the browser.
- * It excludes server-only modules like chatgptScraper, googleAds, and API functions
+ * It excludes server-only modules like llmScraper, googleAds, and API functions
  * that depend on Node.js or Deno-specific APIs.
  *
  * @module

package/esm/browser.js

@@ -2,7 +2,7 @@
  * Browser-safe exports for @graphext/cuery
  *
  * This module only exports types and pure functions that can safely run in the browser.
- * It excludes server-only modules like chatgptScraper, googleAds, and API functions
+ * It excludes server-only modules like llmScraper, googleAds, and API functions
  * that depend on Node.js or Deno-specific APIs.
  *
  * @module

package/esm/mod.d.ts

@@ -5,7 +5,7 @@
  */
 export * from './src/llm.js';
 export { BatchResponse } from './src/response.js';
-export { getProvider, getProviderForModel, getModelPricing, getModelInfo, calculateCost, type ModelPricing, type ModelInfo, } from './src/providers/index.js';
+export { calculateCost, getModelInfo, getModelPricing, getProvider, getProviderForModel, type ModelInfo, type ModelPricing, } from './src/providers/index.js';
 export * from './src/tools/keywords.js';
 export * from './src/tools/classifier.js';
 export * from './src/tools/funnel.js';
@@ -26,8 +26,8 @@ export * from './src/tools/scorer.js';
 export * from './src/helpers/seedKeywords.js';
 export * from './src/tools/generic.js';
 export * from './src/apis/hasdata/index.js';
-export * from './src/apis/brightdata/index.js';
-export * from './src/apis/chatgptScraper/index.js';
+export * from './src/apis/brightdata/contentScraper/index.js';
+export * from './src/apis/brightdata/llmScraper/index.js';
 export * from './src/apis/googleAds/keywordPlanner.js';
 export * from './src/schemas/index.js';
 //# sourceMappingURL=mod.d.ts.map

package/esm/mod.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"mod.d.ts","sourceRoot":"","sources":["../src/mod.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,cAAc,cAAc,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAClD,OAAO,EACN,WAAW,EACX,mBAAmB,EACnB,eAAe,EACf,YAAY,EACZ,aAAa,EACb,KAAK,YAAY,EACjB,KAAK,SAAS,GACd,MAAM,0BAA0B,CAAC;AAGlC,cAAc,yBAAyB,CAAC;AACxC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,uBAAuB,CAAC;AACtC,cAAc,yBAAyB,CAAC;AACxC,cAAc,uBAAuB,CAAC;AACtC,cAAc,uBAAuB,CAAC;AACtC,cAAc,wBAAwB,CAAC;AACvC,cAAc,wBAAwB,CAAC;AACvC,cAAc,uBAAuB,CAAC;AACtC,cAAc,uBAAuB,CAAC;AACtC,cAAc,0BAA0B,CAAC;AACzC,cAAc,0BAA0B,CAAC;AACzC,cAAc,0BAA0B,CAAC;AACzC,cAAc,wBAAwB,CAAC;AACvC,cAAc,yBAAyB,CAAC;AACxC,cAAc,wBAAwB,CAAC;AACvC,cAAc,uBAAuB,CAAC;AACtC,cAAc,+BAA+B,CAAC;AAC9C,cAAc,wBAAwB,CAAC;AACvC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,gCAAgC,CAAC;AAC/C,cAAc,oCAAoC,CAAC;AACnD,cAAc,wCAAwC,CAAC;AACvD,cAAc,wBAAwB,CAAC"}
+{"version":3,"file":"mod.d.ts","sourceRoot":"","sources":["../src/mod.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,cAAc,cAAc,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAClD,OAAO,EACN,aAAa,EACb,YAAY,EACZ,eAAe,EACf,WAAW,EACX,mBAAmB,EACnB,KAAK,SAAS,EACd,KAAK,YAAY,GACjB,MAAM,0BAA0B,CAAC;AAGlC,cAAc,yBAAyB,CAAC;AACxC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,uBAAuB,CAAC;AACtC,cAAc,yBAAyB,CAAC;AACxC,cAAc,uBAAuB,CAAC;AACtC,cAAc,uBAAuB,CAAC;AACtC,cAAc,wBAAwB,CAAC;AACvC,cAAc,wBAAwB,CAAC;AACvC,cAAc,uBAAuB,CAAC;AACtC,cAAc,uBAAuB,CAAC;AACtC,cAAc,0BAA0B,CAAC;AACzC,cAAc,0BAA0B,CAAC;AACzC,cAAc,0BAA0B,CAAC;AACzC,cAAc,wBAAwB,CAAC;AACvC,cAAc,yBAAyB,CAAC;AACxC,cAAc,wBAAwB,CAAC;AACvC,cAAc,uBAAuB,CAAC;AACtC,cAAc,+BAA+B,CAAC;AAC9C,cAAc,wBAAwB,CAAC;AACvC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,+CAA+C,CAAC;AAC9D,cAAc,2CAA2C,CAAC;AAC1D,cAAc,wCAAwC,CAAC;AACvD,cAAc,wBAAwB,CAAC"}

package/esm/mod.js

@@ -6,7 +6,7 @@
 // Core LLM interface and types
 export * from './src/llm.js';
 export { BatchResponse } from './src/response.js';
-export { getProvider, getProviderForModel, getModelPricing, getModelInfo, calculateCost, } from './src/providers/index.js';
+export { calculateCost, getModelInfo, getModelPricing, getProvider, getProviderForModel, } from './src/providers/index.js';
 // Tools
 export * from './src/tools/keywords.js';
 export * from './src/tools/classifier.js';
@@ -28,7 +28,7 @@ export * from './src/tools/scorer.js';
 export * from './src/helpers/seedKeywords.js';
 export * from './src/tools/generic.js';
 export * from './src/apis/hasdata/index.js';
-export * from './src/apis/brightdata/index.js';
-export * from './src/apis/chatgptScraper/index.js';
+export * from './src/apis/brightdata/contentScraper/index.js';
+export * from './src/apis/brightdata/llmScraper/index.js';
 export * from './src/apis/googleAds/keywordPlanner.js';
 export * from './src/schemas/index.js';

package/esm/src/apis/brightdata/contentScraper/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/src/apis/brightdata/contentScraper/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC"}

package/{script/src/apis/brightdata → esm/src/apis/brightdata/contentScraper}/scrape.d.ts

@@ -1,4 +1,4 @@
-import { type RetryConfig } from '../../helpers/async.js';
+import { type RetryConfig } from '../../../helpers/async.js';
 export interface BrightDataScrapeOptions {
     /** BrightData zone name. Defaults to "web_unlocker". */
     zone?: string;

package/esm/src/apis/brightdata/contentScraper/scrape.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scrape.d.ts","sourceRoot":"","sources":["../../../../../src/src/apis/brightdata/contentScraper/scrape.ts"],"names":[],"mappings":"AACA,OAAO,EAA4B,KAAK,WAAW,EAAE,MAAM,2BAA2B,CAAC;AAavF,MAAM,WAAW,uBAAuB;IACpC,wDAAwD;IACxD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,6DAA6D;IAC7D,MAAM,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;IACxB,8DAA8D;IAC9D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,wDAAwD;IACxD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oDAAoD;IACpD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,kCAAkC;IAClC,WAAW,CAAC,EAAE,WAAW,CAAC;CAC7B;AAED,MAAM,WAAW,wBAAwB;IACrC,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;CACjB;AA4ED;;;GAGG;AACH,wBAAsB,gBAAgB,CAClC,GAAG,EAAE,MAAM,EACX,OAAO,GAAE,uBAA4B,GACtC,OAAO,CAAC,wBAAwB,CAAC,CAYnC;AAED;;;GAGG;AACH,wBAAsB,qBAAqB,CACvC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,EACnB,OAAO,GAAE,uBAA4B,EACrC,cAAc,GAAE,MAA+B,GAChD,OAAO,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC,CAQ1C"}

package/esm/src/apis/brightdata/{scrape.js → contentScraper/scrape.js}

@@ -1,5 +1,5 @@
-import * as dntShim from "../../../_dnt.shims.js";
-import { mapParallel, withRetries } from '../../helpers/async.js';
+import * as dntShim from "../../../../_dnt.shims.js";
+import { mapParallel, withRetries } from '../../../helpers/async.js';
 const BRIGHTDATA_CONCURRENCY = 10;
 const BRIGHTDATA_RETRY_CONFIG = {
     maxRetries: 3,

package/esm/src/apis/brightdata/llmScraper/brightdata.d.ts

@@ -0,0 +1,20 @@
+import { type ProviderFunctions } from './scrape.js';
+interface BrightdataProviderConfig {
+    apiBase: string;
+    datasetId: string;
+    outputFields: Array<string>;
+    extraFields?: Array<string>;
+    targetUrl: string;
+    extraInputs?: (params: {
+        prompt: string;
+        useSearch: boolean;
+        countryISOCode: string | null;
+    }) => Record<string, unknown>;
+    providerName: string;
+    maxConcurrency: number;
+    maxPromptsPerRequest: number;
+}
+export declare function createBrightdataProvider(overrides?: Partial<BrightdataProviderConfig>): ProviderFunctions;
+export declare const brightdataProvider: ProviderFunctions;
+export {};
+//# sourceMappingURL=brightdata.d.ts.map

package/esm/src/apis/brightdata/llmScraper/brightdata.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"brightdata.d.ts","sourceRoot":"","sources":["../../../../../src/src/apis/brightdata/llmScraper/brightdata.ts"],"names":[],"mappings":"AAcA,OAAO,EAA6C,KAAK,iBAAiB,EAAE,MAAM,aAAa,CAAC;AA4BhG,UAAU,wBAAwB;IACjC,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAC5B,WAAW,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,CACb,MAAM,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,OAAO,CAAC;QAAC,cAAc,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,KACzE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC7B,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,oBAAoB,EAAE,MAAM,CAAC;CAC7B;AA2DD,wBAAgB,wBAAwB,CACvC,SAAS,GAAE,OAAO,CAAC,wBAAwB,CAAM,GAC/C,iBAAiB,CAgJnB;AAMD,eAAO,MAAM,kBAAkB,EAAE,iBAA8C,CAAC"}

package/esm/src/apis/brightdata/llmScraper/brightdata.js

@@ -0,0 +1,182 @@
+/* eslint no-console: ["warn", { allow: ["log", "warn", "error"] }] */
+/**
+ * Brightdata LLM Scraper Provider.
+ *
+ * API Flow:
+ * 1. Trigger: POST to /datasets/v3/trigger → returns snapshot_id
+ * 2. Monitor: GET /datasets/v3/progress/{snapshot_id} until ready
+ * 3. Download: GET /datasets/v3/snapshot/{snapshot_id}
+ */
+import * as dntShim from "../../../../_dnt.shims.js";
+import { sleep, withRetries } from '../../../helpers/async.js';
+import { buildSources, cleanAnswer, getAbortSignal } from './scrape.js';
+const DEFAULT_BRIGHTDATA_PROVIDER_CONFIG = {
+    apiBase: 'https://api.brightdata.com',
+    datasetId: 'gd_m7aof0k82r803d5bjm',
+    outputFields: [
+        'url',
+        'prompt',
+        'answer_text',
+        'answer_text_markdown',
+        'citations',
+        'links_attached',
+        'country',
+        'index',
+    ],
+    targetUrl: 'http://chatgpt.com/',
+    providerName: 'Brightdata',
+    maxConcurrency: 50,
+    maxPromptsPerRequest: 1,
+};
+const TRIGGER_RETRY = {
+    maxRetries: 3,
+    initialDelay: 0,
+    statusCodes: [429, 500, 502, 503, 504],
+};
+const DOWNLOAD_RETRY = {
+    maxRetries: 5,
+    initialDelay: 2000,
+    statusCodes: [202, 500, 502, 503, 504],
+};
+const MONITOR_RETRY = {
+    maxRetries: 4,
+    initialDelay: 1000,
+    statusCodes: [408, 425, 429, 500, 502, 503, 504],
+};
+const MONITOR_RETRIABLE = new Set(MONITOR_RETRY.statusCodes ?? []);
+const MAX_WAIT_MS = 600_000; // 10 minutes
+const POLL_INTERVAL_MS = 5_000;
+// ============================================================================
+// API Key
+// ============================================================================
+function getApiKey() {
+    const apiKey = dntShim.Deno.env.get('BRIGHTDATA_API_KEY');
+    if (!apiKey) {
+        throw new Error('BRIGHTDATA_API_KEY environment variable is required');
+    }
+    return apiKey;
+}
+// ============================================================================
+// Provider Functions
+// ============================================================================
+export function createBrightdataProvider(overrides = {}) {
+    const config = { ...DEFAULT_BRIGHTDATA_PROVIDER_CONFIG, ...overrides };
+    const customOutputFields = [...new Set([...(config.outputFields ?? []), ...(config.extraFields ?? [])])].join('|');
+    async function triggerJob(prompt, useSearch, countryISOCode) {
+        const apiKey = getApiKey();
+        const url = `${config.apiBase}/datasets/v3/trigger?dataset_id=${config.datasetId}&include_errors=true`;
+        const input = {
+            url: config.targetUrl,
+            prompt,
+            country: countryISOCode || '',
+            index: 0,
+        };
+        Object.assign(input, config.extraInputs?.({ prompt, useSearch, countryISOCode }) ?? {});
+        const body = {
+            custom_output_fields: customOutputFields,
+            input: [input],
+        };
+        try {
+            const response = await withRetries(() => fetch(url, {
+                method: 'POST',
+                headers: {
+                    'Authorization': `Bearer ${apiKey}`,
+                    'Content-Type': 'application/json',
+                },
+                body: JSON.stringify(body),
+                signal: getAbortSignal(),
+            }), TRIGGER_RETRY);
+            if (!response.ok) {
+                console.error(`[${config.providerName}] Trigger error: ${response.status}`);
+                return null;
+            }
+            const data = await response.json();
+            return data?.snapshot_id || null;
+        }
+        catch (error) {
+            console.error(`[${config.providerName}] Trigger failed:`, error);
+            return null;
+        }
+    }
+    async function monitorJob(snapshotId) {
+        const apiKey = getApiKey();
+        const url = `${config.apiBase}/datasets/v3/progress/${snapshotId}`;
+        const startTime = Date.now();
+        const abortSignal = getAbortSignal();
+        while (Date.now() - startTime < MAX_WAIT_MS) {
+            if (abortSignal?.aborted)
+                return false;
+            try {
+                const response = await withRetries(() => fetch(url, {
+                    headers: { 'Authorization': `Bearer ${apiKey}` },
+                    signal: abortSignal,
+                }), MONITOR_RETRY);
+                if (!response.ok) {
+                    if (!MONITOR_RETRIABLE.has(response.status))
+                        return false;
+                }
+                else {
+                    const status = await response.json();
+                    if (status.status === 'ready' || status.status === 'complete')
+                        return true;
+                    if (status.status === 'failed' || status.status === 'error')
+                        return false;
+                }
+            }
+            catch (error) {
+                console.error(`[${config.providerName}] Monitor error:`, error);
+            }
+            await sleep(POLL_INTERVAL_MS, abortSignal);
+        }
+        console.error(`[${config.providerName}] Monitor timeout after ${MAX_WAIT_MS / 1000}s`);
+        return false;
+    }
+    async function downloadJob(snapshotId) {
+        const apiKey = getApiKey();
+        const url = `${config.apiBase}/datasets/v3/snapshot/${snapshotId}?format=json`;
+        try {
+            const response = await withRetries(() => fetch(url, {
+                headers: { 'Authorization': `Bearer ${apiKey}` },
+                signal: getAbortSignal(),
+            }), DOWNLOAD_RETRY);
+            if (!response.ok) {
+                console.error(`[${config.providerName}] Download error: ${response.status}`);
+                return null;
+            }
+            const data = await response.json();
+            return Array.isArray(data) ? data : null;
+        }
+        catch (error) {
+            console.error(`[${config.providerName}] Download failed:`, error);
+            return null;
+        }
+    }
+    function transformResponse(raw) {
+        const responses = raw;
+        if (!responses || responses.length === 0)
+            return null;
+        const response = responses[0];
+        const answerText = cleanAnswer(response.answer_text || '');
+        const answerTextMarkdown = cleanAnswer(response.answer_text_markdown || '');
+        return {
+            prompt: response.prompt,
+            answer: answerText,
+            answer_text_markdown: answerTextMarkdown,
+            sources: buildSources(response.citations ?? [], response.links_attached ?? []),
+            searchQueries: response.web_search_query || [],
+        };
+    }
+    return {
+        name: config.providerName,
+        maxConcurrency: config.maxConcurrency,
+        maxPromptsPerRequest: config.maxPromptsPerRequest,
+        triggerJob,
+        monitorJob,
+        downloadJob,
+        transformResponse,
+    };
+}
+// ============================================================================
+// Export
+// ============================================================================
+export const brightdataProvider = createBrightdataProvider();

package/esm/src/apis/brightdata/llmScraper/index.d.ts

@@ -0,0 +1,14 @@
+import type { ModelResult } from '../../../schemas/models.schema.js';
+import { type BatchOptions } from './scrape.js';
+export type { BatchOptions };
+export type JobId = string | null;
+export type ScraperTarget = 'chatgpt' | 'aim';
+export declare function getMaxConcurrency(target?: ScraperTarget): number;
+export declare function getMaxPromptsPerRequest(target?: ScraperTarget): number;
+export declare function scrapeGPTBatch(options: BatchOptions): Promise<Array<ModelResult>>;
+export declare function triggerGPTBatch(options: BatchOptions): Promise<Array<string | null>>;
+export declare function downloadGPTSnapshots(jobIds: Array<string | null>): Promise<Array<ModelResult>>;
+export declare function scrapeAIMBatch(options: BatchOptions): Promise<Array<ModelResult>>;
+export declare function triggerAIMBatch(options: BatchOptions): Promise<Array<string | null>>;
+export declare function downloadAIMSnapshots(jobIds: Array<string | null>): Promise<Array<ModelResult>>;
+//# sourceMappingURL=index.d.ts.map

package/esm/src/apis/brightdata/llmScraper/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/src/apis/brightdata/llmScraper/index.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mCAAmC,CAAC;AACrE,OAAO,EAAE,KAAK,YAAY,EAAqC,MAAM,aAAa,CAAC;AAKnF,YAAY,EAAE,YAAY,EAAE,CAAC;AAC7B,MAAM,MAAM,KAAK,GAAG,MAAM,GAAG,IAAI,CAAC;AAClC,MAAM,MAAM,aAAa,GAAG,SAAS,GAAG,KAAK,CAAC;AAuE9C,wBAAgB,iBAAiB,CAAC,MAAM,GAAE,aAAyB,GAAG,MAAM,CAE3E;AAED,wBAAgB,uBAAuB,CAAC,MAAM,GAAE,aAAyB,GAAG,MAAM,CAEjF;AAGD,wBAAsB,cAAc,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAEvF;AAED,wBAAsB,eAAe,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAE1F;AAED,wBAAsB,oBAAoB,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAEpG;AAGD,wBAAsB,cAAc,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAEvF;AAED,wBAAsB,eAAe,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAE1F;AAED,wBAAsB,oBAAoB,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAEpG"}

package/esm/src/apis/brightdata/llmScraper/index.js

@@ -0,0 +1,97 @@
+/* eslint no-console: ["warn", { allow: ["log", "warn", "error"] }] */
+/**
+ * LLM Scraper - Public API
+ *
+ * Selects between Brightdata and Oxylabs based on LLM_SCRAPER_PROVIDER env var.
+ * Falls back to CHATGPT_SCRAPER_PROVIDER for backward compatibility.
+ * Exposes ChatGPT and AIM variants while preserving the legacy GPT method names.
+ * Default: oxylabs
+ */
+import * as dntShim from "../../../../_dnt.shims.js";
+import { createLLMScraper } from './scrape.js';
+import { createBrightdataProvider } from './brightdata.js';
+import { createOxylabsProvider } from './oxy.js';
+// ============================================================================
+// Scraper Instance (lazy singleton)
+// ============================================================================
+const scrapers = new Map();
+const CHATGPT_BRIGHTDATA_OPTIONS = {
+    datasetId: 'gd_m7aof0k82r803d5bjm',
+    extraFields: ['web_search_triggered', 'web_search_query'],
+    extraInputs: ({ useSearch }) => ({
+        web_search: useSearch,
+    }),
+    targetUrl: 'http://chatgpt.com/',
+};
+const CHATGPT_OXYLABS_OPTIONS = {
+    source: 'chatgpt',
+};
+const AIM_BRIGHTDATA_OPTIONS = {
+    datasetId: 'gd_mcswdt6z2elth3zqr2',
+    targetUrl: 'https://google.com/aimode',
+};
+const AIM_OXYLABS_OPTIONS = {
+    source: 'google_ai_mode',
+    inputKey: 'query',
+    render: 'html',
+    search: undefined,
+};
+function getProviderName() {
+    return dntShim.Deno.env.get('LLM_SCRAPER_PROVIDER')?.toLowerCase() ??
+        dntShim.Deno.env.get('CHATGPT_SCRAPER_PROVIDER')?.toLowerCase();
+}
+function getTargetOptions(target) {
+    return target === 'aim'
+        ? {
+            brightdata: AIM_BRIGHTDATA_OPTIONS,
+            oxylabs: AIM_OXYLABS_OPTIONS,
+        }
+        : {
+            brightdata: CHATGPT_BRIGHTDATA_OPTIONS,
+            oxylabs: CHATGPT_OXYLABS_OPTIONS,
+        };
+}
+function getLLMScraper(target = 'chatgpt') {
+    const existingScraper = scrapers.get(target);
+    if (existingScraper) {
+        return existingScraper;
+    }
+    const providerName = getProviderName();
+    const targetOptions = getTargetOptions(target);
+    const provider = providerName === 'brightdata'
+        ? createBrightdataProvider(targetOptions.brightdata)
+        : createOxylabsProvider(targetOptions.oxylabs);
+    const scraper = createLLMScraper(provider);
+    scrapers.set(target, scraper);
+    return scraper;
+}
+// ============================================================================
+// Public API
+// ============================================================================
+// Shared scraper metadata
+export function getMaxConcurrency(target = 'chatgpt') {
+    return getLLMScraper(target).maxConcurrency;
+}
+export function getMaxPromptsPerRequest(target = 'chatgpt') {
+    return getLLMScraper(target).maxPromptsPerRequest;
+}
+// ChatGPT scraper methods
+export async function scrapeGPTBatch(options) {
+    return getLLMScraper('chatgpt').scrapeLLMBatch(options);
+}
+export async function triggerGPTBatch(options) {
+    return getLLMScraper('chatgpt').triggerLLMBatch(options);
+}
+export async function downloadGPTSnapshots(jobIds) {
+    return getLLMScraper('chatgpt').downloadLLMSnapshots(jobIds);
+}
+// AIM scraper methods
+export async function scrapeAIMBatch(options) {
+    return getLLMScraper('aim').scrapeLLMBatch(options);
+}
+export async function triggerAIMBatch(options) {
+    return getLLMScraper('aim').triggerLLMBatch(options);
+}
+export async function downloadAIMSnapshots(jobIds) {
+    return getLLMScraper('aim').downloadLLMSnapshots(jobIds);
+}

package/esm/src/apis/brightdata/llmScraper/oxy.d.ts

@@ -0,0 +1,16 @@
+import { type ProviderFunctions } from './scrape.js';
+interface OxylabsProviderConfig {
+    apiBase: string;
+    source: string;
+    inputKey: 'prompt' | 'query';
+    parse: boolean;
+    search?: boolean;
+    render?: 'html';
+    providerName: string;
+    maxConcurrency: number;
+    maxPromptsPerRequest: number;
+}
+export declare function createOxylabsProvider(overrides?: Partial<OxylabsProviderConfig>): ProviderFunctions;
+export declare const oxylabsProvider: ProviderFunctions;
+export {};
+//# sourceMappingURL=oxy.d.ts.map

package/esm/src/apis/brightdata/llmScraper/oxy.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"oxy.d.ts","sourceRoot":"","sources":["../../../../../src/src/apis/brightdata/llmScraper/oxy.ts"],"names":[],"mappings":"AAeA,OAAO,EAA6C,KAAK,iBAAiB,EAAE,MAAM,aAAa,CAAC;AA2BhG,UAAU,qBAAqB;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC;IAC7B,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,oBAAoB,EAAE,MAAM,CAAC;CAC7B;AAyCD,wBAAgB,qBAAqB,CAAC,SAAS,GAAE,OAAO,CAAC,qBAAqB,CAAM,GAAG,iBAAiB,CAyJvG;AAMD,eAAO,MAAM,eAAe,EAAE,iBAA2C,CAAC"}

package/esm/src/apis/brightdata/llmScraper/oxy.js

@@ -0,0 +1,171 @@
+/* eslint no-console: ["warn", { allow: ["log", "warn", "error"] }] */
+/**
+ * Oxylabs LLM Scraper Provider.
+ *
+ * API Flow (Async Push-Pull):
+ * 1. Trigger: POST to /v1/queries → returns job id
+ * 2. Monitor: GET /v1/queries/{id} until status is 'done'
+ * 3. Download: GET /v1/queries/{id}/results
+ */
+import * as dntShim from "../../../../_dnt.shims.js";
+import { sleep, withRetries } from '../../../helpers/async.js';
+import { buildSources, cleanAnswer, getAbortSignal } from './scrape.js';
+const DEFAULT_OXYLABS_PROVIDER_CONFIG = {
+    apiBase: 'https://data.oxylabs.io/v1',
+    source: 'chatgpt',
+    inputKey: 'prompt',
+    parse: true,
+    search: true,
+    providerName: 'Oxylabs',
+    maxConcurrency: 10,
+    maxPromptsPerRequest: 1,
+};
+const RETRY_CONFIG = {
+    maxRetries: 3,
+    initialDelay: 1000,
+    statusCodes: [429, 500, 502, 503, 504, 524, 612, 613],
+};
+const MAX_WAIT_MS = 600_000; // 10 minutes
+const POLL_INTERVAL_MS = 5_000;
+// ============================================================================
+// Auth
+// ============================================================================
+function getAuthHeader() {
+    const username = dntShim.Deno.env.get('OXYLABS_USERNAME');
+    const password = dntShim.Deno.env.get('OXYLABS_PASSWORD');
+    if (!username || !password) {
+        throw new Error('OXYLABS_USERNAME and OXYLABS_PASSWORD environment variables are required');
+    }
+    return `Basic ${btoa(`${username}:${password}`)}`;
+}
+// ============================================================================
+// Provider Functions
+// ============================================================================
+export function createOxylabsProvider(overrides = {}) {
+    const config = { ...DEFAULT_OXYLABS_PROVIDER_CONFIG, ...overrides };
+    async function triggerJob(prompt, _useSearch, countryISOCode) {
+        const authHeader = getAuthHeader();
+        const url = `${config.apiBase}/queries`;
+        const body = {
+            source: config.source,
+            parse: config.parse,
+            [config.inputKey]: prompt,
+        };
+        if (config.search != null) {
+            body.search = config.search;
+        }
+        if (config.render != null) {
+            body.render = config.render;
+        }
+        if (countryISOCode) {
+            body.geo_location = countryISOCode;
+        }
+        try {
+            const response = await withRetries(() => fetch(url, {
+                method: 'POST',
+                headers: {
+                    'Authorization': authHeader,
+                    'Content-Type': 'application/json',
+                },
+                body: JSON.stringify(body),
+                signal: getAbortSignal(),
+            }), RETRY_CONFIG);
+            if (!response.ok) {
+                console.error(`[${config.providerName}] Trigger error: ${response.status}`);
+                return null;
+            }
+            const data = await response.json();
+            return data?.id || null;
+        }
+        catch (error) {
+            console.error(`[${config.providerName}] Trigger failed:`, error);
+            return null;
+        }
+    }
+    async function monitorJob(jobId) {
+        const authHeader = getAuthHeader();
+        const url = `${config.apiBase}/queries/${jobId}`;
+        const startTime = Date.now();
+        const abortSignal = getAbortSignal();
+        while (Date.now() - startTime < MAX_WAIT_MS) {
+            if (abortSignal?.aborted)
+                return false;
+            try {
+                const response = await fetch(url, {
+                    headers: { 'Authorization': authHeader },
+                    signal: abortSignal,
+                });
+                // 204 = job not completed yet, continue polling
+                if (response.status === 204) {
+                    await sleep(POLL_INTERVAL_MS, abortSignal);
+                    continue;
+                }
+                if (response.ok) {
+                    const status = await response.json();
+                    if (status.status === 'done')
+                        return true;
+                    if (status.status === 'faulted' || status.status === 'failed')
+                        return false;
+                }
+            }
+            catch (error) {
+                console.error(`[${config.providerName}] Monitor error:`, error);
+            }
+            await sleep(POLL_INTERVAL_MS, abortSignal);
+        }
+        console.error(`[${config.providerName}] Monitor timeout after ${MAX_WAIT_MS / 1000}s`);
+        return false;
+    }
+    async function downloadJob(jobId) {
+        const authHeader = getAuthHeader();
+        const url = `${config.apiBase}/queries/${jobId}/results`;
+        try {
+            const response = await withRetries(() => fetch(url, {
+                headers: { 'Authorization': authHeader },
+                signal: getAbortSignal(),
+            }), RETRY_CONFIG);
+            if (!response.ok) {
+                console.error(`[${config.providerName}] Download error: ${response.status}`);
+                return null;
+            }
+            return await response.json();
+        }
+        catch (error) {
+            console.error(`[${config.providerName}] Download failed:`, error);
+            return null;
+        }
+    }
+    function transformResponse(raw) {
+        const response = raw;
+        const content = response?.results?.[0]?.content;
+        if (!content)
+            return null;
+        const answerText = cleanAnswer(content.response_text || '');
+        const answerTextMarkdown = cleanAnswer(content.markdown_text || '');
+        // Map section='citations' to cited=true (like Brightdata's cited field)
+        const citations = (content.citations ?? []).map((c) => ({
+            ...c,
+            cited: c.section === 'citations',
+        }));
+        return {
+            prompt: content.prompt || '',
+            answer: answerText,
+            answer_text_markdown: answerTextMarkdown,
+            sources: buildSources(citations),
+            searchQueries: [],
+        };
+    }
+    return {
+        name: config.providerName,
+        maxConcurrency: config.maxConcurrency,
+        maxPromptsPerRequest: config.maxPromptsPerRequest,
+        triggerJob,
+        monitorJob,
+        downloadJob,
+        transformResponse,
+    };
+}
+// ============================================================================
+// Export
+// ============================================================================
+export const oxylabsProvider = createOxylabsProvider();

package/{script/src/apis/chatgptScraper/scraper.d.ts → esm/src/apis/brightdata/llmScraper/scrape.d.ts}

@@ -1,5 +1,5 @@
-import type { ModelResult } from '../../schemas/models.schema.js';
-import type { Source, SearchSource } from '../../schemas/sources.schema.js';
+import type { ModelResult } from '../../../schemas/models.schema.js';
+import type { Source } from '../../../schemas/sources.schema.js';
 export interface BatchOptions {
     prompts: Array<string>;
     useSearch?: boolean;
@@ -14,12 +14,12 @@ export interface ProviderFunctions {
     downloadJob: (jobId: string) => Promise<unknown>;
     transformResponse: (raw: unknown) => ModelResult | null;
 }
-export interface GPTScraper {
+export interface LLMScraper {
     maxConcurrency: number;
     maxPromptsPerRequest: number;
-    scrapeGPTBatch: (options: BatchOptions) => Promise<Array<ModelResult>>;
-    triggerGPTBatch: (options: BatchOptions) => Promise<Array<string | null>>;
-    downloadGPTSnapshots: (jobIds: Array<string | null>) => Promise<Array<ModelResult>>;
+    scrapeLLMBatch: (options: BatchOptions) => Promise<Array<ModelResult>>;
+    triggerLLMBatch: (options: BatchOptions) => Promise<Array<string | null>>;
+    downloadLLMSnapshots: (jobIds: Array<string | null>) => Promise<Array<ModelResult>>;
 }
 export declare function getAbortSignal(): AbortSignal | undefined;
 export declare function cleanAnswer(answer: string): string;
@@ -29,18 +29,15 @@ export declare function buildSources(citations: Array<{
     description?: string;
     text?: string;
     cited?: boolean;
-}>, linkPositions?: Record<string, Array<number>>): Array<Source>;
-export declare function buildSearchSources(sources: Array<{
+}>, linksAttached?: Array<{
     url?: string;
-    title?: string;
-    snippet?: string;
-    rank?: number;
-    date_published?: string;
-}>): Array<SearchSource>;
+    text?: string;
+    position?: number;
+}>): Array<Source>;
 /**
  * Creates an empty model result for failed jobs.
  * This ensures we always return the same number of rows as input.
  */
 export declare function emptyModelResult(providerName: string, errorMessage?: string, context?: unknown): ModelResult;
-export declare function createScraper(provider: ProviderFunctions): GPTScraper;
-//# sourceMappingURL=scraper.d.ts.map
+export declare function createLLMScraper(provider: ProviderFunctions): LLMScraper;
+//# sourceMappingURL=scrape.d.ts.map