@graphext/cuery 0.9.5 → 0.10.1

package/script/src/apis/chatgptScraper/oxy.js

@@ -1,192 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
-    };
-})();
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.oxylabsProvider = void 0;
-/* eslint no-console: ["warn", { allow: ["log", "warn", "error"] }] */
-/**
- * Oxylabs GPT Scraper Provider.
- *
- * API Flow (Async Push-Pull):
- * 1. Trigger: POST to /v1/queries → returns job id
- * 2. Monitor: GET /v1/queries/{id} until status is 'done'
- * 3. Download: GET /v1/queries/{id}/results
- */
-const dntShim = __importStar(require("../../../_dnt.shims.js"));
-const async_js_1 = require("../../helpers/async.js");
-const scraper_js_1 = require("./scraper.js");
-// ============================================================================
-// Constants
-// ============================================================================
-const API_BASE = 'https://data.oxylabs.io/v1';
-const RETRY_CONFIG = {
-    maxRetries: 3,
-    initialDelay: 1000,
-    statusCodes: [429, 500, 502, 503, 504, 524, 612, 613]
-};
-const MAX_WAIT_MS = 600_000; // 10 minutes
-const POLL_INTERVAL_MS = 5_000;
-// ============================================================================
-// Auth
-// ============================================================================
-function getAuthHeader() {
-    const username = dntShim.Deno.env.get('OXYLABS_USERNAME');
-    const password = dntShim.Deno.env.get('OXYLABS_PASSWORD');
-    if (!username || !password) {
-        throw new Error('OXYLABS_USERNAME and OXYLABS_PASSWORD environment variables are required');
-    }
-    return `Basic ${btoa(`${username}:${password}`)}`;
-}
-// ============================================================================
-// Provider Functions
-// ============================================================================
-async function triggerJob(prompt, useSearch, countryISOCode) {
-    const authHeader = getAuthHeader();
-    const url = `${API_BASE}/queries`;
-    const body = {
-        source: 'chatgpt',
-        prompt,
-        parse: true,
-        search: true // Oxylabs requires search: true (cannot be false or blank)
-    };
-    if (countryISOCode) {
-        body.geo_location = countryISOCode;
-    }
-    try {
-        const response = await (0, async_js_1.withRetries)(() => fetch(url, {
-            method: 'POST',
-            headers: {
-                'Authorization': authHeader,
-                'Content-Type': 'application/json'
-            },
-            body: JSON.stringify(body),
-            signal: (0, scraper_js_1.getAbortSignal)()
-        }), RETRY_CONFIG);
-        if (!response.ok) {
-            console.error(`[Oxylabs] Trigger error: ${response.status}`);
-            return null;
-        }
-        const data = await response.json();
-        return data?.id || null;
-    }
-    catch (error) {
-        console.error('[Oxylabs] Trigger failed:', error);
-        return null;
-    }
-}
-async function monitorJob(jobId) {
-    const authHeader = getAuthHeader();
-    const url = `${API_BASE}/queries/${jobId}`;
-    const startTime = Date.now();
-    const abortSignal = (0, scraper_js_1.getAbortSignal)();
-    while (Date.now() - startTime < MAX_WAIT_MS) {
-        if (abortSignal?.aborted)
-            return false;
-        try {
-            const response = await fetch(url, {
-                headers: { 'Authorization': authHeader },
-                signal: abortSignal
-            });
-            // 204 = job not completed yet, continue polling
-            if (response.status === 204) {
-                await (0, async_js_1.sleep)(POLL_INTERVAL_MS, abortSignal);
-                continue;
-            }
-            if (response.ok) {
-                const status = await response.json();
-                if (status.status === 'done')
-                    return true;
-                if (status.status === 'faulted' || status.status === 'failed')
-                    return false;
-            }
-        }
-        catch (error) {
-            console.error('[Oxylabs] Monitor error:', error);
-        }
-        await (0, async_js_1.sleep)(POLL_INTERVAL_MS, abortSignal);
-    }
-    console.error(`[Oxylabs] Monitor timeout after ${MAX_WAIT_MS / 1000}s`);
-    return false;
-}
-async function downloadJob(jobId) {
-    const authHeader = getAuthHeader();
-    const url = `${API_BASE}/queries/${jobId}/results`;
-    try {
-        const response = await (0, async_js_1.withRetries)(() => fetch(url, {
-            headers: { 'Authorization': authHeader },
-            signal: (0, scraper_js_1.getAbortSignal)()
-        }), RETRY_CONFIG);
-        if (!response.ok) {
-            console.error(`[Oxylabs] Download error: ${response.status}`);
-            return null;
-        }
-        return await response.json();
-    }
-    catch (error) {
-        console.error('[Oxylabs] Download failed:', error);
-        return null;
-    }
-}
-function transformResponse(raw) {
-    const response = raw;
-    const content = response?.results?.[0]?.content;
-    if (!content)
-        return null;
-    let answer = content.markdown_text || content.response_text || '';
-    answer = (0, scraper_js_1.cleanAnswer)(answer);
-    // Map section='citations' to cited=true (like Brightdata's cited field)
-    const citations = (content.citations ?? []).map(c => ({
-        ...c,
-        cited: c.section === 'citations'
-    }));
-    return {
-        prompt: content.prompt || '',
-        answer,
-        sources: (0, scraper_js_1.buildSources)(citations),
-        searchQueries: [],
-        searchSources: []
-    };
-}
-// ============================================================================
-// Export
-// ============================================================================
-exports.oxylabsProvider = {
-    name: 'Oxylabs',
-    maxConcurrency: 10,
-    maxPromptsPerRequest: 1,
-    triggerJob,
-    monitorJob,
-    downloadJob,
-    transformResponse
-};

package/script/src/apis/chatgptScraper/scraper.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"scraper.d.ts","sourceRoot":"","sources":["../../../../src/src/apis/chatgptScraper/scraper.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,gCAAgC,CAAC;AAClE,OAAO,KAAK,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,iCAAiC,CAAC;AAO5E,MAAM,WAAW,YAAY;IAC5B,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACvB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/B;AAED,MAAM,WAAW,iBAAiB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,EAAE,MAAM,CAAC;IACvB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,UAAU,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,GAAG,IAAI,KAAK,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAC1G,UAAU,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IAChD,WAAW,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IACjD,iBAAiB,EAAE,CAAC,GAAG,EAAE,OAAO,KAAK,WAAW,GAAG,IAAI,CAAC;CACxD;AAED,MAAM,WAAW,UAAU;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,cAAc,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC;IACvE,eAAe,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC;IAC1E,oBAAoB,EAAE,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC;CACpF;AAMD,wBAAgB,cAAc,IAAI,WAAW,GAAG,SAAS,CAExD;AAED,wBAAgB,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAMlD;AAED,wBAAgB,YAAY,CAC3B,SAAS,EAAE,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,OAAO,CAAA;CAAE,CAAC,EACvG,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,GAC3C,KAAK,CAAC,MAAM,CAAC,CAQf;AAED,wBAAgB,kBAAkB,CACjC,OAAO,EAAE,KAAK,CAAC;IAAE,GAAG,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,cAAc,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,GACxG,KAAK,CAAC,YAAY,CAAC,CAQrB;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,YAAY,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,GAAG,WAAW,CAS5G;AAMD,wBAAgB,aAAa,CAAC,QAAQ,EAAE,iBAAiB,GAAG,UAAU,CAkErE"}

package/script/src/apis/chatgptScraper/scraper.js

@@ -1,139 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
-    };
-})();
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.getAbortSignal = getAbortSignal;
-exports.cleanAnswer = cleanAnswer;
-exports.buildSources = buildSources;
-exports.buildSearchSources = buildSearchSources;
-exports.emptyModelResult = emptyModelResult;
-exports.createScraper = createScraper;
-/* eslint no-console: ["warn", { allow: ["log", "warn", "error"] }] */
-/**
- * GPT Scraper - Core types and orchestration logic.
- *
- * Uses composition: providers supply functions, this module orchestrates them.
- */
-const dntShim = __importStar(require("../../../_dnt.shims.js"));
-const async_js_1 = require("../../helpers/async.js");
-const urls_js_1 = require("../../helpers/urls.js");
-// ============================================================================
-// Shared Utilities
-// ============================================================================
-function getAbortSignal() {
-    return dntShim.dntGlobalThis.abortSignal;
-}
-function cleanAnswer(answer) {
-    return answer
-        .replace(/!\[([^\]]*)\]\([^)]+\)/g, '')
-        .replace(/\n\s*Image\s*\n/g, '\n')
-        .replace(/\n{3,}/g, '\n\n')
-        .trim();
-}
-function buildSources(citations, linkPositions) {
-    return citations.map(c => ({
-        title: c.title || c.description || c.text || '',
-        url: c.url,
-        domain: (0, urls_js_1.extractDomain)(c.url),
-        cited: c.cited,
-        positions: linkPositions?.[c.url]
-    }));
-}
-function buildSearchSources(sources) {
-    return sources.map(s => ({
-        title: s.title || s.snippet || '',
-        url: s.url || '',
-        domain: s.url ? (0, urls_js_1.extractDomain)(s.url) : '',
-        rank: s.rank || 0,
-        datePublished: s.date_published || null
-    }));
-}
-/**
- * Creates an empty model result for failed jobs.
- * This ensures we always return the same number of rows as input.
- */
-function emptyModelResult(providerName, errorMessage, context) {
-    if (errorMessage) {
-        console.error(`[${providerName}] ${errorMessage}`, context ?? '');
-    }
-    return {
-        prompt: '',
-        answer: '',
-        sources: []
-    };
-}
-// ============================================================================
-// Scraper Factory
-// ============================================================================
-function createScraper(provider) {
-    const { name, maxConcurrency, maxPromptsPerRequest, triggerJob, monitorJob, downloadJob, transformResponse } = provider;
-    async function triggerGPTBatch({ prompts, useSearch = false, countryISOCode = null }) {
-        const jobIds = await (0, async_js_1.mapParallel)(prompts, maxConcurrency, (prompt) => triggerJob(prompt, useSearch, countryISOCode));
-        console.log(`[${name}] Triggered ${jobIds.length} jobs for ${prompts.length} prompts`);
-        return jobIds;
-    }
-    async function downloadGPTSnapshots(jobIds) {
-        const results = [];
-        for (const jobId of jobIds) {
-            if (!jobId) {
-                results.push(emptyModelResult(name, 'No job ID provided'));
-                continue;
-            }
-            const isReady = await monitorJob(jobId);
-            if (!isReady) {
-                results.push(emptyModelResult(name, 'Job not ready or failed', jobId));
-                continue;
-            }
-            const raw = await downloadJob(jobId);
-            if (!raw) {
-                results.push(emptyModelResult(name, 'Failed to download job', jobId));
-                continue;
-            }
-            const result = transformResponse(raw);
-            results.push(result ?? emptyModelResult(name, 'Failed to transform response', jobId));
-        }
-        return results;
-    }
-    async function scrapeGPTBatch(options) {
-        const jobIds = await triggerGPTBatch(options);
-        return downloadGPTSnapshots(jobIds);
-    }
-    return {
-        maxConcurrency,
-        maxPromptsPerRequest,
-        scrapeGPTBatch,
-        triggerGPTBatch,
-        downloadGPTSnapshots
-    };
-}