@graphext/cuery 0.9.4 → 0.10.0

package/script/src/apis/chatgptScraper/brightdata.js

@@ -1,208 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
-    };
-})();
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.brightdataProvider = void 0;
-/* eslint no-console: ["warn", { allow: ["log", "warn", "error"] }] */
-/**
- * Brightdata GPT Scraper Provider.
- *
- * API Flow:
- * 1. Trigger: POST to /datasets/v3/trigger → returns snapshot_id
- * 2. Monitor: GET /datasets/v3/progress/{snapshot_id} until ready
- * 3. Download: GET /datasets/v3/snapshot/{snapshot_id}
- */
-const dntShim = __importStar(require("../../../_dnt.shims.js"));
-const async_js_1 = require("../../helpers/async.js");
-const scraper_js_1 = require("./scraper.js");
-// ============================================================================
-// Constants
-// ============================================================================
-const API_BASE = 'https://api.brightdata.com';
-const DATASET_ID = 'gd_m7aof0k82r803d5bjm';
-const OUTPUT_FIELDS = 'url|prompt|answer_text|answer_text_markdown|citations|links_attached|search_sources|country|model|web_search_triggered|web_search_query|index';
-const TRIGGER_RETRY = {
-    maxRetries: 3,
-    initialDelay: 0,
-    statusCodes: [429, 500, 502, 503, 504]
-};
-const DOWNLOAD_RETRY = {
-    maxRetries: 5,
-    initialDelay: 2000,
-    statusCodes: [202, 500, 502, 503, 504]
-};
-const MONITOR_RETRY = {
-    maxRetries: 4,
-    initialDelay: 1000,
-    statusCodes: [408, 425, 429, 500, 502, 503, 504]
-};
-const MONITOR_RETRIABLE = new Set(MONITOR_RETRY.statusCodes ?? []);
-const MAX_WAIT_MS = 600_000; // 10 minutes
-const POLL_INTERVAL_MS = 5_000;
-// ============================================================================
-// API Key
-// ============================================================================
-function getApiKey() {
-    const apiKey = dntShim.Deno.env.get('BRIGHTDATA_API_KEY');
-    if (!apiKey) {
-        throw new Error('BRIGHTDATA_API_KEY environment variable is required');
-    }
-    return apiKey;
-}
-// ============================================================================
-// Provider Functions
-// ============================================================================
-async function triggerJob(prompt, useSearch, countryISOCode) {
-    const apiKey = getApiKey();
-    const url = `${API_BASE}/datasets/v3/trigger?dataset_id=${DATASET_ID}&include_errors=true`;
-    const body = {
-        custom_output_fields: OUTPUT_FIELDS,
-        input: [{
-                url: 'http://chatgpt.com/',
-                prompt,
-                web_search: useSearch,
-                country: countryISOCode || '',
-                index: 0
-            }]
-    };
-    try {
-        const response = await (0, async_js_1.withRetries)(() => fetch(url, {
-            method: 'POST',
-            headers: {
-                'Authorization': `Bearer ${apiKey}`,
-                'Content-Type': 'application/json'
-            },
-            body: JSON.stringify(body),
-            signal: (0, scraper_js_1.getAbortSignal)()
-        }), TRIGGER_RETRY);
-        if (!response.ok) {
-            console.error(`[Brightdata] Trigger error: ${response.status}`);
-            return null;
-        }
-        const data = await response.json();
-        return data?.snapshot_id || null;
-    }
-    catch (error) {
-        console.error('[Brightdata] Trigger failed:', error);
-        return null;
-    }
-}
-async function monitorJob(snapshotId) {
-    const apiKey = getApiKey();
-    const url = `${API_BASE}/datasets/v3/progress/${snapshotId}`;
-    const startTime = Date.now();
-    const abortSignal = (0, scraper_js_1.getAbortSignal)();
-    while (Date.now() - startTime < MAX_WAIT_MS) {
-        if (abortSignal?.aborted)
-            return false;
-        try {
-            const response = await (0, async_js_1.withRetries)(() => fetch(url, {
-                headers: { 'Authorization': `Bearer ${apiKey}` },
-                signal: abortSignal
-            }), MONITOR_RETRY);
-            if (!response.ok) {
-                if (!MONITOR_RETRIABLE.has(response.status))
-                    return false;
-            }
-            else {
-                const status = await response.json();
-                if (status.status === 'ready' || status.status === 'complete')
-                    return true;
-                if (status.status === 'failed' || status.status === 'error')
-                    return false;
-            }
-        }
-        catch (error) {
-            console.error('[Brightdata] Monitor error:', error);
-        }
-        await (0, async_js_1.sleep)(POLL_INTERVAL_MS, abortSignal);
-    }
-    console.error(`[Brightdata] Monitor timeout after ${MAX_WAIT_MS / 1000}s`);
-    return false;
-}
-async function downloadJob(snapshotId) {
-    const apiKey = getApiKey();
-    const url = `${API_BASE}/datasets/v3/snapshot/${snapshotId}?format=json`;
-    try {
-        const response = await (0, async_js_1.withRetries)(() => fetch(url, {
-            headers: { 'Authorization': `Bearer ${apiKey}` },
-            signal: (0, scraper_js_1.getAbortSignal)()
-        }), DOWNLOAD_RETRY);
-        if (!response.ok) {
-            console.error(`[Brightdata] Download error: ${response.status}`);
-            return null;
-        }
-        const data = await response.json();
-        return Array.isArray(data) ? data : null;
-    }
-    catch (error) {
-        console.error('[Brightdata] Download failed:', error);
-        return null;
-    }
-}
-function transformResponse(raw) {
-    const responses = raw;
-    if (!responses || responses.length === 0)
-        return null;
-    const response = responses[0];
-    let answer = response.answer_text_markdown || response.answer_text || '';
-    answer = (0, scraper_js_1.cleanAnswer)(answer);
-    // Build link positions map
-    const linkPositions = {};
-    for (const link of response.links_attached ?? []) {
-        if (link.url && link.position != null) {
-            linkPositions[link.url] ??= [];
-            linkPositions[link.url].push(link.position);
-        }
-    }
-    return {
-        prompt: response.prompt,
-        answer,
-        sources: (0, scraper_js_1.buildSources)(response.citations ?? [], linkPositions),
-        searchQueries: response.web_search_query || [],
-        searchSources: (0, scraper_js_1.buildSearchSources)(response.search_sources ?? [])
-    };
-}
-// ============================================================================
-// Export
-// ============================================================================
-exports.brightdataProvider = {
-    name: 'Brightdata',
-    maxConcurrency: 50,
-    maxPromptsPerRequest: 1,
-    triggerJob,
-    monitorJob,
-    downloadJob,
-    transformResponse
-};

package/script/src/apis/chatgptScraper/index.d.ts

@@ -1,10 +0,0 @@
-import type { ModelResult } from '../../schemas/models.schema.js';
-import { type BatchOptions } from './scraper.js';
-export type { BatchOptions };
-export type JobId = string | null;
-export declare function getMaxConcurrency(): number;
-export declare function getMaxPromptsPerRequest(): number;
-export declare function scrapeGPTBatch(options: BatchOptions): Promise<Array<ModelResult>>;
-export declare function triggerGPTBatch(options: BatchOptions): Promise<Array<string | null>>;
-export declare function downloadGPTSnapshots(jobIds: Array<string | null>): Promise<Array<ModelResult>>;
-//# sourceMappingURL=index.d.ts.map

package/script/src/apis/chatgptScraper/index.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/src/apis/chatgptScraper/index.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,gCAAgC,CAAC;AAClE,OAAO,EAAE,KAAK,YAAY,EAAkC,MAAM,cAAc,CAAC;AAKjF,YAAY,EAAE,YAAY,EAAE,CAAC;AAC7B,MAAM,MAAM,KAAK,GAAG,MAAM,GAAG,IAAI,CAAC;AAqBlC,wBAAgB,iBAAiB,IAAI,MAAM,CAE1C;AAED,wBAAgB,uBAAuB,IAAI,MAAM,CAEhD;AAED,wBAAsB,cAAc,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAEvF;AAED,wBAAsB,eAAe,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAE1F;AAED,wBAAsB,oBAAoB,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAEpG"}

package/script/src/apis/chatgptScraper/index.js

@@ -1,81 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
-    };
-})();
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.getMaxConcurrency = getMaxConcurrency;
-exports.getMaxPromptsPerRequest = getMaxPromptsPerRequest;
-exports.scrapeGPTBatch = scrapeGPTBatch;
-exports.triggerGPTBatch = triggerGPTBatch;
-exports.downloadGPTSnapshots = downloadGPTSnapshots;
-/* eslint no-console: ["warn", { allow: ["log", "warn", "error"] }] */
-/**
- * GPT Scraper - Public API
- *
- * Selects between Brightdata and Oxylabs based on CHATGPT_SCRAPER_PROVIDER env var.
- * Default: oxylabs
- */
-const dntShim = __importStar(require("../../../_dnt.shims.js"));
-const scraper_js_1 = require("./scraper.js");
-const brightdata_js_1 = require("./brightdata.js");
-const oxy_js_1 = require("./oxy.js");
-// ============================================================================
-// Scraper Instance (lazy singleton)
-// ============================================================================
-let scraper = null;
-function getScraper() {
-    if (!scraper) {
-        const providerName = dntShim.Deno.env.get('CHATGPT_SCRAPER_PROVIDER')?.toLowerCase();
-        const provider = providerName === 'brightdata' ? brightdata_js_1.brightdataProvider : oxy_js_1.oxylabsProvider;
-        scraper = (0, scraper_js_1.createScraper)(provider);
-    }
-    return scraper;
-}
-// ============================================================================
-// Public API
-// ============================================================================
-function getMaxConcurrency() {
-    return getScraper().maxConcurrency;
-}
-function getMaxPromptsPerRequest() {
-    return getScraper().maxPromptsPerRequest;
-}
-async function scrapeGPTBatch(options) {
-    return getScraper().scrapeGPTBatch(options);
-}
-async function triggerGPTBatch(options) {
-    return getScraper().triggerGPTBatch(options);
-}
-async function downloadGPTSnapshots(jobIds) {
-    return getScraper().downloadGPTSnapshots(jobIds);
-}

package/script/src/apis/chatgptScraper/oxy.d.ts

@@ -1,3 +0,0 @@
-import { type ProviderFunctions } from './scraper.js';
-export declare const oxylabsProvider: ProviderFunctions;
-//# sourceMappingURL=oxy.d.ts.map

package/script/src/apis/chatgptScraper/oxy.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"oxy.d.ts","sourceRoot":"","sources":["../../../../src/src/apis/chatgptScraper/oxy.ts"],"names":[],"mappings":"AAeA,OAAO,EACN,KAAK,iBAAiB,EAItB,MAAM,cAAc,CAAC;AA6LtB,eAAO,MAAM,eAAe,EAAE,iBAQ7B,CAAC"}

package/script/src/apis/chatgptScraper/oxy.js

@@ -1,192 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
-    };
-})();
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.oxylabsProvider = void 0;
-/* eslint no-console: ["warn", { allow: ["log", "warn", "error"] }] */
-/**
- * Oxylabs GPT Scraper Provider.
- *
- * API Flow (Async Push-Pull):
- * 1. Trigger: POST to /v1/queries → returns job id
- * 2. Monitor: GET /v1/queries/{id} until status is 'done'
- * 3. Download: GET /v1/queries/{id}/results
- */
-const dntShim = __importStar(require("../../../_dnt.shims.js"));
-const async_js_1 = require("../../helpers/async.js");
-const scraper_js_1 = require("./scraper.js");
-// ============================================================================
-// Constants
-// ============================================================================
-const API_BASE = 'https://data.oxylabs.io/v1';
-const RETRY_CONFIG = {
-    maxRetries: 3,
-    initialDelay: 1000,
-    statusCodes: [429, 500, 502, 503, 504, 524, 612, 613]
-};
-const MAX_WAIT_MS = 600_000; // 10 minutes
-const POLL_INTERVAL_MS = 5_000;
-// ============================================================================
-// Auth
-// ============================================================================
-function getAuthHeader() {
-    const username = dntShim.Deno.env.get('OXYLABS_USERNAME');
-    const password = dntShim.Deno.env.get('OXYLABS_PASSWORD');
-    if (!username || !password) {
-        throw new Error('OXYLABS_USERNAME and OXYLABS_PASSWORD environment variables are required');
-    }
-    return `Basic ${btoa(`${username}:${password}`)}`;
-}
-// ============================================================================
-// Provider Functions
-// ============================================================================
-async function triggerJob(prompt, useSearch, countryISOCode) {
-    const authHeader = getAuthHeader();
-    const url = `${API_BASE}/queries`;
-    const body = {
-        source: 'chatgpt',
-        prompt,
-        parse: true,
-        search: true // Oxylabs requires search: true (cannot be false or blank)
-    };
-    if (countryISOCode) {
-        body.geo_location = countryISOCode;
-    }
-    try {
-        const response = await (0, async_js_1.withRetries)(() => fetch(url, {
-            method: 'POST',
-            headers: {
-                'Authorization': authHeader,
-                'Content-Type': 'application/json'
-            },
-            body: JSON.stringify(body),
-            signal: (0, scraper_js_1.getAbortSignal)()
-        }), RETRY_CONFIG);
-        if (!response.ok) {
-            console.error(`[Oxylabs] Trigger error: ${response.status}`);
-            return null;
-        }
-        const data = await response.json();
-        return data?.id || null;
-    }
-    catch (error) {
-        console.error('[Oxylabs] Trigger failed:', error);
-        return null;
-    }
-}
-async function monitorJob(jobId) {
-    const authHeader = getAuthHeader();
-    const url = `${API_BASE}/queries/${jobId}`;
-    const startTime = Date.now();
-    const abortSignal = (0, scraper_js_1.getAbortSignal)();
-    while (Date.now() - startTime < MAX_WAIT_MS) {
-        if (abortSignal?.aborted)
-            return false;
-        try {
-            const response = await fetch(url, {
-                headers: { 'Authorization': authHeader },
-                signal: abortSignal
-            });
-            // 204 = job not completed yet, continue polling
-            if (response.status === 204) {
-                await (0, async_js_1.sleep)(POLL_INTERVAL_MS, abortSignal);
-                continue;
-            }
-            if (response.ok) {
-                const status = await response.json();
-                if (status.status === 'done')
-                    return true;
-                if (status.status === 'faulted' || status.status === 'failed')
-                    return false;
-            }
-        }
-        catch (error) {
-            console.error('[Oxylabs] Monitor error:', error);
-        }
-        await (0, async_js_1.sleep)(POLL_INTERVAL_MS, abortSignal);
-    }
-    console.error(`[Oxylabs] Monitor timeout after ${MAX_WAIT_MS / 1000}s`);
-    return false;
-}
-async function downloadJob(jobId) {
-    const authHeader = getAuthHeader();
-    const url = `${API_BASE}/queries/${jobId}/results`;
-    try {
-        const response = await (0, async_js_1.withRetries)(() => fetch(url, {
-            headers: { 'Authorization': authHeader },
-            signal: (0, scraper_js_1.getAbortSignal)()
-        }), RETRY_CONFIG);
-        if (!response.ok) {
-            console.error(`[Oxylabs] Download error: ${response.status}`);
-            return null;
-        }
-        return await response.json();
-    }
-    catch (error) {
-        console.error('[Oxylabs] Download failed:', error);
-        return null;
-    }
-}
-function transformResponse(raw) {
-    const response = raw;
-    const content = response?.results?.[0]?.content;
-    if (!content)
-        return null;
-    let answer = content.markdown_text || content.response_text || '';
-    answer = (0, scraper_js_1.cleanAnswer)(answer);
-    // Map section='citations' to cited=true (like Brightdata's cited field)
-    const citations = (content.citations ?? []).map(c => ({
-        ...c,
-        cited: c.section === 'citations'
-    }));
-    return {
-        prompt: content.prompt || '',
-        answer,
-        sources: (0, scraper_js_1.buildSources)(citations),
-        searchQueries: [],
-        searchSources: []
-    };
-}
-// ============================================================================
-// Export
-// ============================================================================
-exports.oxylabsProvider = {
-    name: 'Oxylabs',
-    maxConcurrency: 10,
-    maxPromptsPerRequest: 1,
-    triggerJob,
-    monitorJob,
-    downloadJob,
-    transformResponse
-};

package/script/src/apis/chatgptScraper/scraper.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"scraper.d.ts","sourceRoot":"","sources":["../../../../src/src/apis/chatgptScraper/scraper.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,gCAAgC,CAAC;AAClE,OAAO,KAAK,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAC;AAO5E,MAAM,WAAW,YAAY;IAC5B,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACvB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/B;AAED,MAAM,WAAW,iBAAiB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,EAAE,MAAM,CAAC;IACvB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,UAAU,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,GAAG,IAAI,KAAK,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAC1G,UAAU,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IAChD,WAAW,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IACjD,iBAAiB,EAAE,CAAC,GAAG,EAAE,OAAO,KAAK,WAAW,GAAG,IAAI,CAAC;CACxD;AAED,MAAM,WAAW,UAAU;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,cAAc,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC;IACvE,eAAe,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC;IAC1E,oBAAoB,EAAE,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC;CACpF;AAMD,wBAAgB,cAAc,IAAI,WAAW,GAAG,SAAS,CAExD;AAED,wBAAgB,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAMlD;AAED,wBAAgB,YAAY,CAC3B,SAAS,EAAE,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,OAAO,CAAA;CAAE,CAAC,EACvG,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,GAC3C,KAAK,CAAC,MAAM,CAAC,CAQf;AAED,wBAAgB,kBAAkB,CACjC,OAAO,EAAE,KAAK,CAAC;IAAE,GAAG,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,cAAc,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,GACxG,KAAK,CAAC,YAAY,CAAC,CAQrB;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,YAAY,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,GAAG,WAAW,CAS5G;AAMD,wBAAgB,aAAa,CAAC,QAAQ,EAAE,iBAAiB,GAAG,UAAU,CAkErE"}

package/script/src/apis/chatgptScraper/scraper.js

@@ -1,139 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
-    };
-})();
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.getAbortSignal = getAbortSignal;
-exports.cleanAnswer = cleanAnswer;
-exports.buildSources = buildSources;
-exports.buildSearchSources = buildSearchSources;
-exports.emptyModelResult = emptyModelResult;
-exports.createScraper = createScraper;
-/* eslint no-console: ["warn", { allow: ["log", "warn", "error"] }] */
-/**
- * GPT Scraper - Core types and orchestration logic.
- *
- * Uses composition: providers supply functions, this module orchestrates them.
- */
-const dntShim = __importStar(require("../../../_dnt.shims.js"));
-const async_js_1 = require("../../helpers/async.js");
-const urls_js_1 = require("../../helpers/urls.js");
-// ============================================================================
-// Shared Utilities
-// ============================================================================
-function getAbortSignal() {
-    return dntShim.dntGlobalThis.abortSignal;
-}
-function cleanAnswer(answer) {
-    return answer
-        .replace(/!\[([^\]]*)\]\([^)]+\)/g, '')
-        .replace(/\n\s*Image\s*\n/g, '\n')
-        .replace(/\n{3,}/g, '\n\n')
-        .trim();
-}
-function buildSources(citations, linkPositions) {
-    return citations.map(c => ({
-        title: c.title || c.description || c.text || '',
-        url: c.url,
-        domain: (0, urls_js_1.extractDomain)(c.url),
-        cited: c.cited,
-        positions: linkPositions?.[c.url]
-    }));
-}
-function buildSearchSources(sources) {
-    return sources.map(s => ({
-        title: s.title || s.snippet || '',
-        url: s.url || '',
-        domain: s.url ? (0, urls_js_1.extractDomain)(s.url) : '',
-        rank: s.rank || 0,
-        datePublished: s.date_published || null
-    }));
-}
-/**
- * Creates an empty model result for failed jobs.
- * This ensures we always return the same number of rows as input.
- */
-function emptyModelResult(providerName, errorMessage, context) {
-    if (errorMessage) {
-        console.error(`[${providerName}] ${errorMessage}`, context ?? '');
-    }
-    return {
-        prompt: '',
-        answer: '',
-        sources: []
-    };
-}
-// ============================================================================
-// Scraper Factory
-// ============================================================================
-function createScraper(provider) {
-    const { name, maxConcurrency, maxPromptsPerRequest, triggerJob, monitorJob, downloadJob, transformResponse } = provider;
-    async function triggerGPTBatch({ prompts, useSearch = false, countryISOCode = null }) {
-        const jobIds = await (0, async_js_1.mapParallel)(prompts, maxConcurrency, (prompt) => triggerJob(prompt, useSearch, countryISOCode));
-        console.log(`[${name}] Triggered ${jobIds.length} jobs for ${prompts.length} prompts`);
-        return jobIds;
-    }
-    async function downloadGPTSnapshots(jobIds) {
-        const results = [];
-        for (const jobId of jobIds) {
-            if (!jobId) {
-                results.push(emptyModelResult(name, 'No job ID provided'));
-                continue;
-            }
-            const isReady = await monitorJob(jobId);
-            if (!isReady) {
-                results.push(emptyModelResult(name, 'Job not ready or failed', jobId));
-                continue;
-            }
-            const raw = await downloadJob(jobId);
-            if (!raw) {
-                results.push(emptyModelResult(name, 'Failed to download job', jobId));
-                continue;
-            }
-            const result = transformResponse(raw);
-            results.push(result ?? emptyModelResult(name, 'Failed to transform response', jobId));
-        }
-        return results;
-    }
-    async function scrapeGPTBatch(options) {
-        const jobIds = await triggerGPTBatch(options);
-        return downloadGPTSnapshots(jobIds);
-    }
-    return {
-        maxConcurrency,
-        maxPromptsPerRequest,
-        scrapeGPTBatch,
-        triggerGPTBatch,
-        downloadGPTSnapshots
-    };
-}