@graphext/cuery 0.9.5 → 0.10.1

package/script/src/apis/brightdata/llmScraper/brightdata.js

@@ -0,0 +1,219 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.brightdataProvider = void 0;
+exports.createBrightdataProvider = createBrightdataProvider;
+/* eslint no-console: ["warn", { allow: ["log", "warn", "error"] }] */
+/**
+ * Brightdata LLM Scraper Provider.
+ *
+ * API Flow:
+ * 1. Trigger: POST to /datasets/v3/trigger → returns snapshot_id
+ * 2. Monitor: GET /datasets/v3/progress/{snapshot_id} until ready
+ * 3. Download: GET /datasets/v3/snapshot/{snapshot_id}
+ */
+const dntShim = __importStar(require("../../../../_dnt.shims.js"));
+const async_js_1 = require("../../../helpers/async.js");
+const scrape_js_1 = require("./scrape.js");
+const DEFAULT_BRIGHTDATA_PROVIDER_CONFIG = {
+    apiBase: 'https://api.brightdata.com',
+    datasetId: 'gd_m7aof0k82r803d5bjm',
+    outputFields: [
+        'url',
+        'prompt',
+        'answer_text',
+        'answer_text_markdown',
+        'citations',
+        'links_attached',
+        'country',
+        'index',
+    ],
+    targetUrl: 'http://chatgpt.com/',
+    providerName: 'Brightdata',
+    maxConcurrency: 50,
+    maxPromptsPerRequest: 1,
+};
+const TRIGGER_RETRY = {
+    maxRetries: 3,
+    initialDelay: 0,
+    statusCodes: [429, 500, 502, 503, 504],
+};
+const DOWNLOAD_RETRY = {
+    maxRetries: 5,
+    initialDelay: 2000,
+    statusCodes: [202, 500, 502, 503, 504],
+};
+const MONITOR_RETRY = {
+    maxRetries: 4,
+    initialDelay: 1000,
+    statusCodes: [408, 425, 429, 500, 502, 503, 504],
+};
+const MONITOR_RETRIABLE = new Set(MONITOR_RETRY.statusCodes ?? []);
+const MAX_WAIT_MS = 600_000; // 10 minutes
+const POLL_INTERVAL_MS = 5_000;
+// ============================================================================
+// API Key
+// ============================================================================
+function getApiKey() {
+    const apiKey = dntShim.Deno.env.get('BRIGHTDATA_API_KEY');
+    if (!apiKey) {
+        throw new Error('BRIGHTDATA_API_KEY environment variable is required');
+    }
+    return apiKey;
+}
+// ============================================================================
+// Provider Functions
+// ============================================================================
+function createBrightdataProvider(overrides = {}) {
+    const config = { ...DEFAULT_BRIGHTDATA_PROVIDER_CONFIG, ...overrides };
+    const customOutputFields = [...new Set([...(config.outputFields ?? []), ...(config.extraFields ?? [])])].join('|');
+    async function triggerJob(prompt, useSearch, countryISOCode) {
+        const apiKey = getApiKey();
+        const url = `${config.apiBase}/datasets/v3/trigger?dataset_id=${config.datasetId}&include_errors=true`;
+        const input = {
+            url: config.targetUrl,
+            prompt,
+            country: countryISOCode || '',
+            index: 0,
+        };
+        Object.assign(input, config.extraInputs?.({ prompt, useSearch, countryISOCode }) ?? {});
+        const body = {
+            custom_output_fields: customOutputFields,
+            input: [input],
+        };
+        try {
+            const response = await (0, async_js_1.withRetries)(() => fetch(url, {
+                method: 'POST',
+                headers: {
+                    'Authorization': `Bearer ${apiKey}`,
+                    'Content-Type': 'application/json',
+                },
+                body: JSON.stringify(body),
+                signal: (0, scrape_js_1.getAbortSignal)(),
+            }), TRIGGER_RETRY);
+            if (!response.ok) {
+                console.error(`[${config.providerName}] Trigger error: ${response.status}`);
+                return null;
+            }
+            const data = await response.json();
+            return data?.snapshot_id || null;
+        }
+        catch (error) {
+            console.error(`[${config.providerName}] Trigger failed:`, error);
+            return null;
+        }
+    }
+    async function monitorJob(snapshotId) {
+        const apiKey = getApiKey();
+        const url = `${config.apiBase}/datasets/v3/progress/${snapshotId}`;
+        const startTime = Date.now();
+        const abortSignal = (0, scrape_js_1.getAbortSignal)();
+        while (Date.now() - startTime < MAX_WAIT_MS) {
+            if (abortSignal?.aborted)
+                return false;
+            try {
+                const response = await (0, async_js_1.withRetries)(() => fetch(url, {
+                    headers: { 'Authorization': `Bearer ${apiKey}` },
+                    signal: abortSignal,
+                }), MONITOR_RETRY);
+                if (!response.ok) {
+                    if (!MONITOR_RETRIABLE.has(response.status))
+                        return false;
+                }
+                else {
+                    const status = await response.json();
+                    if (status.status === 'ready' || status.status === 'complete')
+                        return true;
+                    if (status.status === 'failed' || status.status === 'error')
+                        return false;
+                }
+            }
+            catch (error) {
+                console.error(`[${config.providerName}] Monitor error:`, error);
+            }
+            await (0, async_js_1.sleep)(POLL_INTERVAL_MS, abortSignal);
+        }
+        console.error(`[${config.providerName}] Monitor timeout after ${MAX_WAIT_MS / 1000}s`);
+        return false;
+    }
+    async function downloadJob(snapshotId) {
+        const apiKey = getApiKey();
+        const url = `${config.apiBase}/datasets/v3/snapshot/${snapshotId}?format=json`;
+        try {
+            const response = await (0, async_js_1.withRetries)(() => fetch(url, {
+                headers: { 'Authorization': `Bearer ${apiKey}` },
+                signal: (0, scrape_js_1.getAbortSignal)(),
+            }), DOWNLOAD_RETRY);
+            if (!response.ok) {
+                console.error(`[${config.providerName}] Download error: ${response.status}`);
+                return null;
+            }
+            const data = await response.json();
+            return Array.isArray(data) ? data : null;
+        }
+        catch (error) {
+            console.error(`[${config.providerName}] Download failed:`, error);
+            return null;
+        }
+    }
+    function transformResponse(raw) {
+        const responses = raw;
+        if (!responses || responses.length === 0)
+            return null;
+        const response = responses[0];
+        const answerText = (0, scrape_js_1.cleanAnswer)(response.answer_text || '');
+        const answerTextMarkdown = (0, scrape_js_1.cleanAnswer)(response.answer_text_markdown || '');
+        return {
+            prompt: response.prompt,
+            answer: answerText,
+            answer_text_markdown: answerTextMarkdown,
+            sources: (0, scrape_js_1.buildSources)(response.citations ?? [], response.links_attached ?? []),
+            searchQueries: response.web_search_query || [],
+        };
+    }
+    return {
+        name: config.providerName,
+        maxConcurrency: config.maxConcurrency,
+        maxPromptsPerRequest: config.maxPromptsPerRequest,
+        triggerJob,
+        monitorJob,
+        downloadJob,
+        transformResponse,
+    };
+}
+// ============================================================================
+// Export
+// ============================================================================
+exports.brightdataProvider = createBrightdataProvider();

package/script/src/apis/brightdata/llmScraper/index.d.ts

@@ -0,0 +1,14 @@
+import type { ModelResult } from '../../../schemas/models.schema.js';
+import { type BatchOptions } from './scrape.js';
+export type { BatchOptions };
+export type JobId = string | null;
+export type ScraperTarget = 'chatgpt' | 'aim';
+export declare function getMaxConcurrency(target?: ScraperTarget): number;
+export declare function getMaxPromptsPerRequest(target?: ScraperTarget): number;
+export declare function scrapeGPTBatch(options: BatchOptions): Promise<Array<ModelResult>>;
+export declare function triggerGPTBatch(options: BatchOptions): Promise<Array<string | null>>;
+export declare function downloadGPTSnapshots(jobIds: Array<string | null>): Promise<Array<ModelResult>>;
+export declare function scrapeAIMBatch(options: BatchOptions): Promise<Array<ModelResult>>;
+export declare function triggerAIMBatch(options: BatchOptions): Promise<Array<string | null>>;
+export declare function downloadAIMSnapshots(jobIds: Array<string | null>): Promise<Array<ModelResult>>;
+//# sourceMappingURL=index.d.ts.map

package/script/src/apis/brightdata/llmScraper/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/src/apis/brightdata/llmScraper/index.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mCAAmC,CAAC;AACrE,OAAO,EAAE,KAAK,YAAY,EAAqC,MAAM,aAAa,CAAC;AAKnF,YAAY,EAAE,YAAY,EAAE,CAAC;AAC7B,MAAM,MAAM,KAAK,GAAG,MAAM,GAAG,IAAI,CAAC;AAClC,MAAM,MAAM,aAAa,GAAG,SAAS,GAAG,KAAK,CAAC;AAuE9C,wBAAgB,iBAAiB,CAAC,MAAM,GAAE,aAAyB,GAAG,MAAM,CAE3E;AAED,wBAAgB,uBAAuB,CAAC,MAAM,GAAE,aAAyB,GAAG,MAAM,CAEjF;AAGD,wBAAsB,cAAc,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAEvF;AAED,wBAAsB,eAAe,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAE1F;AAED,wBAAsB,oBAAoB,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAEpG;AAGD,wBAAsB,cAAc,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAEvF;AAED,wBAAsB,eAAe,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAE1F;AAED,wBAAsB,oBAAoB,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAEpG"}

package/script/src/apis/brightdata/llmScraper/index.js

@@ -0,0 +1,140 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getMaxConcurrency = getMaxConcurrency;
+exports.getMaxPromptsPerRequest = getMaxPromptsPerRequest;
+exports.scrapeGPTBatch = scrapeGPTBatch;
+exports.triggerGPTBatch = triggerGPTBatch;
+exports.downloadGPTSnapshots = downloadGPTSnapshots;
+exports.scrapeAIMBatch = scrapeAIMBatch;
+exports.triggerAIMBatch = triggerAIMBatch;
+exports.downloadAIMSnapshots = downloadAIMSnapshots;
+/* eslint no-console: ["warn", { allow: ["log", "warn", "error"] }] */
+/**
+ * LLM Scraper - Public API
+ *
+ * Selects between Brightdata and Oxylabs based on LLM_SCRAPER_PROVIDER env var.
+ * Falls back to CHATGPT_SCRAPER_PROVIDER for backward compatibility.
+ * Exposes ChatGPT and AIM variants while preserving the legacy GPT method names.
+ * Default: oxylabs
+ */
+const dntShim = __importStar(require("../../../../_dnt.shims.js"));
+const scrape_js_1 = require("./scrape.js");
+const brightdata_js_1 = require("./brightdata.js");
+const oxy_js_1 = require("./oxy.js");
+// ============================================================================
+// Scraper Instance (lazy singleton)
+// ============================================================================
+const scrapers = new Map();
+const CHATGPT_BRIGHTDATA_OPTIONS = {
+    datasetId: 'gd_m7aof0k82r803d5bjm',
+    extraFields: ['web_search_triggered', 'web_search_query'],
+    extraInputs: ({ useSearch }) => ({
+        web_search: useSearch,
+    }),
+    targetUrl: 'http://chatgpt.com/',
+};
+const CHATGPT_OXYLABS_OPTIONS = {
+    source: 'chatgpt',
+};
+const AIM_BRIGHTDATA_OPTIONS = {
+    datasetId: 'gd_mcswdt6z2elth3zqr2',
+    targetUrl: 'https://google.com/aimode',
+};
+const AIM_OXYLABS_OPTIONS = {
+    source: 'google_ai_mode',
+    inputKey: 'query',
+    render: 'html',
+    search: undefined,
+};
+function getProviderName() {
+    return dntShim.Deno.env.get('LLM_SCRAPER_PROVIDER')?.toLowerCase() ??
+        dntShim.Deno.env.get('CHATGPT_SCRAPER_PROVIDER')?.toLowerCase();
+}
+function getTargetOptions(target) {
+    return target === 'aim'
+        ? {
+            brightdata: AIM_BRIGHTDATA_OPTIONS,
+            oxylabs: AIM_OXYLABS_OPTIONS,
+        }
+        : {
+            brightdata: CHATGPT_BRIGHTDATA_OPTIONS,
+            oxylabs: CHATGPT_OXYLABS_OPTIONS,
+        };
+}
+function getLLMScraper(target = 'chatgpt') {
+    const existingScraper = scrapers.get(target);
+    if (existingScraper) {
+        return existingScraper;
+    }
+    const providerName = getProviderName();
+    const targetOptions = getTargetOptions(target);
+    const provider = providerName === 'brightdata'
+        ? (0, brightdata_js_1.createBrightdataProvider)(targetOptions.brightdata)
+        : (0, oxy_js_1.createOxylabsProvider)(targetOptions.oxylabs);
+    const scraper = (0, scrape_js_1.createLLMScraper)(provider);
+    scrapers.set(target, scraper);
+    return scraper;
+}
+// ============================================================================
+// Public API
+// ============================================================================
+// Shared scraper metadata
+function getMaxConcurrency(target = 'chatgpt') {
+    return getLLMScraper(target).maxConcurrency;
+}
+function getMaxPromptsPerRequest(target = 'chatgpt') {
+    return getLLMScraper(target).maxPromptsPerRequest;
+}
+// ChatGPT scraper methods
+async function scrapeGPTBatch(options) {
+    return getLLMScraper('chatgpt').scrapeLLMBatch(options);
+}
+async function triggerGPTBatch(options) {
+    return getLLMScraper('chatgpt').triggerLLMBatch(options);
+}
+async function downloadGPTSnapshots(jobIds) {
+    return getLLMScraper('chatgpt').downloadLLMSnapshots(jobIds);
+}
+// AIM scraper methods
+async function scrapeAIMBatch(options) {
+    return getLLMScraper('aim').scrapeLLMBatch(options);
+}
+async function triggerAIMBatch(options) {
+    return getLLMScraper('aim').triggerLLMBatch(options);
+}
+async function downloadAIMSnapshots(jobIds) {
+    return getLLMScraper('aim').downloadLLMSnapshots(jobIds);
+}

package/script/src/apis/brightdata/llmScraper/oxy.d.ts

@@ -0,0 +1,16 @@
+import { type ProviderFunctions } from './scrape.js';
+interface OxylabsProviderConfig {
+    apiBase: string;
+    source: string;
+    inputKey: 'prompt' | 'query';
+    parse: boolean;
+    search?: boolean;
+    render?: 'html';
+    providerName: string;
+    maxConcurrency: number;
+    maxPromptsPerRequest: number;
+}
+export declare function createOxylabsProvider(overrides?: Partial<OxylabsProviderConfig>): ProviderFunctions;
+export declare const oxylabsProvider: ProviderFunctions;
+export {};
+//# sourceMappingURL=oxy.d.ts.map

package/script/src/apis/brightdata/llmScraper/oxy.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"oxy.d.ts","sourceRoot":"","sources":["../../../../../src/src/apis/brightdata/llmScraper/oxy.ts"],"names":[],"mappings":"AAeA,OAAO,EAA6C,KAAK,iBAAiB,EAAE,MAAM,aAAa,CAAC;AA2BhG,UAAU,qBAAqB;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC;IAC7B,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,oBAAoB,EAAE,MAAM,CAAC;CAC7B;AAyCD,wBAAgB,qBAAqB,CAAC,SAAS,GAAE,OAAO,CAAC,qBAAqB,CAAM,GAAG,iBAAiB,CAyJvG;AAMD,eAAO,MAAM,eAAe,EAAE,iBAA2C,CAAC"}

package/script/src/apis/brightdata/llmScraper/oxy.js

@@ -0,0 +1,208 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.oxylabsProvider = void 0;
+exports.createOxylabsProvider = createOxylabsProvider;
+/* eslint no-console: ["warn", { allow: ["log", "warn", "error"] }] */
+/**
+ * Oxylabs LLM Scraper Provider.
+ *
+ * API Flow (Async Push-Pull):
+ * 1. Trigger: POST to /v1/queries → returns job id
+ * 2. Monitor: GET /v1/queries/{id} until status is 'done'
+ * 3. Download: GET /v1/queries/{id}/results
+ */
+const dntShim = __importStar(require("../../../../_dnt.shims.js"));
+const async_js_1 = require("../../../helpers/async.js");
+const scrape_js_1 = require("./scrape.js");
+const DEFAULT_OXYLABS_PROVIDER_CONFIG = {
+    apiBase: 'https://data.oxylabs.io/v1',
+    source: 'chatgpt',
+    inputKey: 'prompt',
+    parse: true,
+    search: true,
+    providerName: 'Oxylabs',
+    maxConcurrency: 10,
+    maxPromptsPerRequest: 1,
+};
+const RETRY_CONFIG = {
+    maxRetries: 3,
+    initialDelay: 1000,
+    statusCodes: [429, 500, 502, 503, 504, 524, 612, 613],
+};
+const MAX_WAIT_MS = 600_000; // 10 minutes
+const POLL_INTERVAL_MS = 5_000;
+// ============================================================================
+// Auth
+// ============================================================================
+function getAuthHeader() {
+    const username = dntShim.Deno.env.get('OXYLABS_USERNAME');
+    const password = dntShim.Deno.env.get('OXYLABS_PASSWORD');
+    if (!username || !password) {
+        throw new Error('OXYLABS_USERNAME and OXYLABS_PASSWORD environment variables are required');
+    }
+    return `Basic ${btoa(`${username}:${password}`)}`;
+}
+// ============================================================================
+// Provider Functions
+// ============================================================================
+function createOxylabsProvider(overrides = {}) {
+    const config = { ...DEFAULT_OXYLABS_PROVIDER_CONFIG, ...overrides };
+    async function triggerJob(prompt, _useSearch, countryISOCode) {
+        const authHeader = getAuthHeader();
+        const url = `${config.apiBase}/queries`;
+        const body = {
+            source: config.source,
+            parse: config.parse,
+            [config.inputKey]: prompt,
+        };
+        if (config.search != null) {
+            body.search = config.search;
+        }
+        if (config.render != null) {
+            body.render = config.render;
+        }
+        if (countryISOCode) {
+            body.geo_location = countryISOCode;
+        }
+        try {
+            const response = await (0, async_js_1.withRetries)(() => fetch(url, {
+                method: 'POST',
+                headers: {
+                    'Authorization': authHeader,
+                    'Content-Type': 'application/json',
+                },
+                body: JSON.stringify(body),
+                signal: (0, scrape_js_1.getAbortSignal)(),
+            }), RETRY_CONFIG);
+            if (!response.ok) {
+                console.error(`[${config.providerName}] Trigger error: ${response.status}`);
+                return null;
+            }
+            const data = await response.json();
+            return data?.id || null;
+        }
+        catch (error) {
+            console.error(`[${config.providerName}] Trigger failed:`, error);
+            return null;
+        }
+    }
+    async function monitorJob(jobId) {
+        const authHeader = getAuthHeader();
+        const url = `${config.apiBase}/queries/${jobId}`;
+        const startTime = Date.now();
+        const abortSignal = (0, scrape_js_1.getAbortSignal)();
+        while (Date.now() - startTime < MAX_WAIT_MS) {
+            if (abortSignal?.aborted)
+                return false;
+            try {
+                const response = await fetch(url, {
+                    headers: { 'Authorization': authHeader },
+                    signal: abortSignal,
+                });
+                // 204 = job not completed yet, continue polling
+                if (response.status === 204) {
+                    await (0, async_js_1.sleep)(POLL_INTERVAL_MS, abortSignal);
+                    continue;
+                }
+                if (response.ok) {
+                    const status = await response.json();
+                    if (status.status === 'done')
+                        return true;
+                    if (status.status === 'faulted' || status.status === 'failed')
+                        return false;
+                }
+            }
+            catch (error) {
+                console.error(`[${config.providerName}] Monitor error:`, error);
+            }
+            await (0, async_js_1.sleep)(POLL_INTERVAL_MS, abortSignal);
+        }
+        console.error(`[${config.providerName}] Monitor timeout after ${MAX_WAIT_MS / 1000}s`);
+        return false;
+    }
+    async function downloadJob(jobId) {
+        const authHeader = getAuthHeader();
+        const url = `${config.apiBase}/queries/${jobId}/results`;
+        try {
+            const response = await (0, async_js_1.withRetries)(() => fetch(url, {
+                headers: { 'Authorization': authHeader },
+                signal: (0, scrape_js_1.getAbortSignal)(),
+            }), RETRY_CONFIG);
+            if (!response.ok) {
+                console.error(`[${config.providerName}] Download error: ${response.status}`);
+                return null;
+            }
+            return await response.json();
+        }
+        catch (error) {
+            console.error(`[${config.providerName}] Download failed:`, error);
+            return null;
+        }
+    }
+    function transformResponse(raw) {
+        const response = raw;
+        const content = response?.results?.[0]?.content;
+        if (!content)
+            return null;
+        const answerText = (0, scrape_js_1.cleanAnswer)(content.response_text || '');
+        const answerTextMarkdown = (0, scrape_js_1.cleanAnswer)(content.markdown_text || '');
+        // Map section='citations' to cited=true (like Brightdata's cited field)
+        const citations = (content.citations ?? []).map((c) => ({
+            ...c,
+            cited: c.section === 'citations',
+        }));
+        return {
+            prompt: content.prompt || '',
+            answer: answerText,
+            answer_text_markdown: answerTextMarkdown,
+            sources: (0, scrape_js_1.buildSources)(citations),
+            searchQueries: [],
+        };
+    }
+    return {
+        name: config.providerName,
+        maxConcurrency: config.maxConcurrency,
+        maxPromptsPerRequest: config.maxPromptsPerRequest,
+        triggerJob,
+        monitorJob,
+        downloadJob,
+        transformResponse,
+    };
+}
+// ============================================================================
+// Export
+// ============================================================================
+exports.oxylabsProvider = createOxylabsProvider();

package/{esm/src/apis/chatgptScraper/scraper.d.ts → script/src/apis/brightdata/llmScraper/scrape.d.ts}

@@ -1,5 +1,5 @@
-import type { ModelResult } from '../../schemas/models.schema.js';
-import type { Source, SearchSource } from '../../schemas/sources.schema.js';
+import type { ModelResult } from '../../../schemas/models.schema.js';
+import type { Source } from '../../../schemas/sources.schema.js';
 export interface BatchOptions {
     prompts: Array<string>;
     useSearch?: boolean;
@@ -14,12 +14,12 @@ export interface ProviderFunctions {
     downloadJob: (jobId: string) => Promise<unknown>;
     transformResponse: (raw: unknown) => ModelResult | null;
 }
-export interface GPTScraper {
+export interface LLMScraper {
     maxConcurrency: number;
     maxPromptsPerRequest: number;
-    scrapeGPTBatch: (options: BatchOptions) => Promise<Array<ModelResult>>;
-    triggerGPTBatch: (options: BatchOptions) => Promise<Array<string | null>>;
-    downloadGPTSnapshots: (jobIds: Array<string | null>) => Promise<Array<ModelResult>>;
+    scrapeLLMBatch: (options: BatchOptions) => Promise<Array<ModelResult>>;
+    triggerLLMBatch: (options: BatchOptions) => Promise<Array<string | null>>;
+    downloadLLMSnapshots: (jobIds: Array<string | null>) => Promise<Array<ModelResult>>;
 }
 export declare function getAbortSignal(): AbortSignal | undefined;
 export declare function cleanAnswer(answer: string): string;
@@ -29,18 +29,15 @@ export declare function buildSources(citations: Array<{
     description?: string;
     text?: string;
     cited?: boolean;
-}>, linkPositions?: Record<string, Array<number>>): Array<Source>;
-export declare function buildSearchSources(sources: Array<{
+}>, linksAttached?: Array<{
     url?: string;
-    title?: string;
-    snippet?: string;
-    rank?: number;
-    date_published?: string;
-}>): Array<SearchSource>;
+    text?: string;
+    position?: number;
+}>): Array<Source>;
 /**
  * Creates an empty model result for failed jobs.
  * This ensures we always return the same number of rows as input.
  */
 export declare function emptyModelResult(providerName: string, errorMessage?: string, context?: unknown): ModelResult;
-export declare function createScraper(provider: ProviderFunctions): GPTScraper;
-//# sourceMappingURL=scraper.d.ts.map
+export declare function createLLMScraper(provider: ProviderFunctions): LLMScraper;
+//# sourceMappingURL=scrape.d.ts.map

package/script/src/apis/brightdata/llmScraper/scrape.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scrape.d.ts","sourceRoot":"","sources":["../../../../../src/src/apis/brightdata/llmScraper/scrape.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mCAAmC,CAAC;AACrE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oCAAoC,CAAC;AAOjE,MAAM,WAAW,YAAY;IAC5B,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACvB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/B;AAED,MAAM,WAAW,iBAAiB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,EAAE,MAAM,CAAC;IACvB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,UAAU,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,GAAG,IAAI,KAAK,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAC1G,UAAU,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IAChD,WAAW,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IACjD,iBAAiB,EAAE,CAAC,GAAG,EAAE,OAAO,KAAK,WAAW,GAAG,IAAI,CAAC;CACxD;AAED,MAAM,WAAW,UAAU;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,cAAc,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC;IACvE,eAAe,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC;IAC1E,oBAAoB,EAAE,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC;CACpF;AAMD,wBAAgB,cAAc,IAAI,WAAW,GAAG,SAAS,CAExD;AAED,wBAAgB,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAMlD;AA+BD,wBAAgB,YAAY,CAC3B,SAAS,EAAE,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,OAAO,CAAA;CAAE,CAAC,EACvG,aAAa,GAAE,KAAK,CAAC;IAAE,GAAG,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,CAAM,GAC3E,KAAK,CAAC,MAAM,CAAC,CAoFf;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,YAAY,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,GAAG,WAAW,CAU5G;AAMD,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,iBAAiB,GAAG,UAAU,CAkExE"}