@graphext/cuery 0.9.4 → 0.10.0

package/script/src/apis/brightdata/llmScraper/index.js

@@ -0,0 +1,140 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getMaxConcurrency = getMaxConcurrency;
+exports.getMaxPromptsPerRequest = getMaxPromptsPerRequest;
+exports.scrapeGPTBatch = scrapeGPTBatch;
+exports.triggerGPTBatch = triggerGPTBatch;
+exports.downloadGPTSnapshots = downloadGPTSnapshots;
+exports.scrapeAIMBatch = scrapeAIMBatch;
+exports.triggerAIMBatch = triggerAIMBatch;
+exports.downloadAIMSnapshots = downloadAIMSnapshots;
+/* eslint no-console: ["warn", { allow: ["log", "warn", "error"] }] */
+/**
+ * LLM Scraper - Public API
+ *
+ * Selects between Brightdata and Oxylabs based on LLM_SCRAPER_PROVIDER env var.
+ * Falls back to CHATGPT_SCRAPER_PROVIDER for backward compatibility.
+ * Exposes ChatGPT and AIM variants while preserving the legacy GPT method names.
+ * Default: oxylabs
+ */
+const dntShim = __importStar(require("../../../../_dnt.shims.js"));
+const scrape_js_1 = require("./scrape.js");
+const brightdata_js_1 = require("./brightdata.js");
+const oxy_js_1 = require("./oxy.js");
+// ============================================================================
+// Scraper Instance (lazy singleton)
+// ============================================================================
+const scrapers = new Map();
+const CHATGPT_BRIGHTDATA_OPTIONS = {
+    datasetId: 'gd_m7aof0k82r803d5bjm',
+    extraFields: ['web_search_triggered', 'web_search_query'],
+    extraInputs: ({ useSearch }) => ({
+        web_search: useSearch,
+    }),
+    targetUrl: 'http://chatgpt.com/',
+};
+const CHATGPT_OXYLABS_OPTIONS = {
+    source: 'chatgpt',
+};
+const AIM_BRIGHTDATA_OPTIONS = {
+    datasetId: 'gd_mcswdt6z2elth3zqr2',
+    targetUrl: 'https://google.com/aimode',
+};
+const AIM_OXYLABS_OPTIONS = {
+    source: 'google_ai_mode',
+    inputKey: 'query',
+    render: 'html',
+    search: undefined,
+};
+function getProviderName() {
+    return dntShim.Deno.env.get('LLM_SCRAPER_PROVIDER')?.toLowerCase() ??
+        dntShim.Deno.env.get('CHATGPT_SCRAPER_PROVIDER')?.toLowerCase();
+}
+function getTargetOptions(target) {
+    return target === 'aim'
+        ? {
+            brightdata: AIM_BRIGHTDATA_OPTIONS,
+            oxylabs: AIM_OXYLABS_OPTIONS,
+        }
+        : {
+            brightdata: CHATGPT_BRIGHTDATA_OPTIONS,
+            oxylabs: CHATGPT_OXYLABS_OPTIONS,
+        };
+}
+function getLLMScraper(target = 'chatgpt') {
+    const existingScraper = scrapers.get(target);
+    if (existingScraper) {
+        return existingScraper;
+    }
+    const providerName = getProviderName();
+    const targetOptions = getTargetOptions(target);
+    const provider = providerName === 'brightdata'
+        ? (0, brightdata_js_1.createBrightdataProvider)(targetOptions.brightdata)
+        : (0, oxy_js_1.createOxylabsProvider)(targetOptions.oxylabs);
+    const scraper = (0, scrape_js_1.createLLMScraper)(provider);
+    scrapers.set(target, scraper);
+    return scraper;
+}
+// ============================================================================
+// Public API
+// ============================================================================
+// Shared scraper metadata
+function getMaxConcurrency(target = 'chatgpt') {
+    return getLLMScraper(target).maxConcurrency;
+}
+function getMaxPromptsPerRequest(target = 'chatgpt') {
+    return getLLMScraper(target).maxPromptsPerRequest;
+}
+// ChatGPT scraper methods
+async function scrapeGPTBatch(options) {
+    return getLLMScraper('chatgpt').scrapeLLMBatch(options);
+}
+async function triggerGPTBatch(options) {
+    return getLLMScraper('chatgpt').triggerLLMBatch(options);
+}
+async function downloadGPTSnapshots(jobIds) {
+    return getLLMScraper('chatgpt').downloadLLMSnapshots(jobIds);
+}
+// AIM scraper methods
+async function scrapeAIMBatch(options) {
+    return getLLMScraper('aim').scrapeLLMBatch(options);
+}
+async function triggerAIMBatch(options) {
+    return getLLMScraper('aim').triggerLLMBatch(options);
+}
+async function downloadAIMSnapshots(jobIds) {
+    return getLLMScraper('aim').downloadLLMSnapshots(jobIds);
+}

package/script/src/apis/brightdata/llmScraper/oxy.d.ts

@@ -0,0 +1,16 @@
+import { type ProviderFunctions } from './scrape.js';
+interface OxylabsProviderConfig {
+    apiBase: string;
+    source: string;
+    inputKey: 'prompt' | 'query';
+    parse: boolean;
+    search?: boolean;
+    render?: 'html';
+    providerName: string;
+    maxConcurrency: number;
+    maxPromptsPerRequest: number;
+}
+export declare function createOxylabsProvider(overrides?: Partial<OxylabsProviderConfig>): ProviderFunctions;
+export declare const oxylabsProvider: ProviderFunctions;
+export {};
+//# sourceMappingURL=oxy.d.ts.map

package/script/src/apis/brightdata/llmScraper/oxy.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"oxy.d.ts","sourceRoot":"","sources":["../../../../../src/src/apis/brightdata/llmScraper/oxy.ts"],"names":[],"mappings":"AAeA,OAAO,EAA6C,KAAK,iBAAiB,EAAE,MAAM,aAAa,CAAC;AA2BhG,UAAU,qBAAqB;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC;IAC7B,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,oBAAoB,EAAE,MAAM,CAAC;CAC7B;AAyCD,wBAAgB,qBAAqB,CAAC,SAAS,GAAE,OAAO,CAAC,qBAAqB,CAAM,GAAG,iBAAiB,CAyJvG;AAMD,eAAO,MAAM,eAAe,EAAE,iBAA2C,CAAC"}

package/script/src/apis/brightdata/llmScraper/oxy.js

@@ -0,0 +1,208 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.oxylabsProvider = void 0;
+exports.createOxylabsProvider = createOxylabsProvider;
+/* eslint no-console: ["warn", { allow: ["log", "warn", "error"] }] */
+/**
+ * Oxylabs LLM Scraper Provider.
+ *
+ * API Flow (Async Push-Pull):
+ * 1. Trigger: POST to /v1/queries → returns job id
+ * 2. Monitor: GET /v1/queries/{id} until status is 'done'
+ * 3. Download: GET /v1/queries/{id}/results
+ */
+const dntShim = __importStar(require("../../../../_dnt.shims.js"));
+const async_js_1 = require("../../../helpers/async.js");
+const scrape_js_1 = require("./scrape.js");
+const DEFAULT_OXYLABS_PROVIDER_CONFIG = {
+    apiBase: 'https://data.oxylabs.io/v1',
+    source: 'chatgpt',
+    inputKey: 'prompt',
+    parse: true,
+    search: true,
+    providerName: 'Oxylabs',
+    maxConcurrency: 10,
+    maxPromptsPerRequest: 1,
+};
+const RETRY_CONFIG = {
+    maxRetries: 3,
+    initialDelay: 1000,
+    statusCodes: [429, 500, 502, 503, 504, 524, 612, 613],
+};
+const MAX_WAIT_MS = 600_000; // 10 minutes
+const POLL_INTERVAL_MS = 5_000;
+// ============================================================================
+// Auth
+// ============================================================================
+function getAuthHeader() {
+    const username = dntShim.Deno.env.get('OXYLABS_USERNAME');
+    const password = dntShim.Deno.env.get('OXYLABS_PASSWORD');
+    if (!username || !password) {
+        throw new Error('OXYLABS_USERNAME and OXYLABS_PASSWORD environment variables are required');
+    }
+    return `Basic ${btoa(`${username}:${password}`)}`;
+}
+// ============================================================================
+// Provider Functions
+// ============================================================================
+function createOxylabsProvider(overrides = {}) {
+    const config = { ...DEFAULT_OXYLABS_PROVIDER_CONFIG, ...overrides };
+    async function triggerJob(prompt, _useSearch, countryISOCode) {
+        const authHeader = getAuthHeader();
+        const url = `${config.apiBase}/queries`;
+        const body = {
+            source: config.source,
+            parse: config.parse,
+            [config.inputKey]: prompt,
+        };
+        if (config.search != null) {
+            body.search = config.search;
+        }
+        if (config.render != null) {
+            body.render = config.render;
+        }
+        if (countryISOCode) {
+            body.geo_location = countryISOCode;
+        }
+        try {
+            const response = await (0, async_js_1.withRetries)(() => fetch(url, {
+                method: 'POST',
+                headers: {
+                    'Authorization': authHeader,
+                    'Content-Type': 'application/json',
+                },
+                body: JSON.stringify(body),
+                signal: (0, scrape_js_1.getAbortSignal)(),
+            }), RETRY_CONFIG);
+            if (!response.ok) {
+                console.error(`[${config.providerName}] Trigger error: ${response.status}`);
+                return null;
+            }
+            const data = await response.json();
+            return data?.id || null;
+        }
+        catch (error) {
+            console.error(`[${config.providerName}] Trigger failed:`, error);
+            return null;
+        }
+    }
+    async function monitorJob(jobId) {
+        const authHeader = getAuthHeader();
+        const url = `${config.apiBase}/queries/${jobId}`;
+        const startTime = Date.now();
+        const abortSignal = (0, scrape_js_1.getAbortSignal)();
+        while (Date.now() - startTime < MAX_WAIT_MS) {
+            if (abortSignal?.aborted)
+                return false;
+            try {
+                const response = await fetch(url, {
+                    headers: { 'Authorization': authHeader },
+                    signal: abortSignal,
+                });
+                // 204 = job not completed yet, continue polling
+                if (response.status === 204) {
+                    await (0, async_js_1.sleep)(POLL_INTERVAL_MS, abortSignal);
+                    continue;
+                }
+                if (response.ok) {
+                    const status = await response.json();
+                    if (status.status === 'done')
+                        return true;
+                    if (status.status === 'faulted' || status.status === 'failed')
+                        return false;
+                }
+            }
+            catch (error) {
+                console.error(`[${config.providerName}] Monitor error:`, error);
+            }
+            await (0, async_js_1.sleep)(POLL_INTERVAL_MS, abortSignal);
+        }
+        console.error(`[${config.providerName}] Monitor timeout after ${MAX_WAIT_MS / 1000}s`);
+        return false;
+    }
+    async function downloadJob(jobId) {
+        const authHeader = getAuthHeader();
+        const url = `${config.apiBase}/queries/${jobId}/results`;
+        try {
+            const response = await (0, async_js_1.withRetries)(() => fetch(url, {
+                headers: { 'Authorization': authHeader },
+                signal: (0, scrape_js_1.getAbortSignal)(),
+            }), RETRY_CONFIG);
+            if (!response.ok) {
+                console.error(`[${config.providerName}] Download error: ${response.status}`);
+                return null;
+            }
+            return await response.json();
+        }
+        catch (error) {
+            console.error(`[${config.providerName}] Download failed:`, error);
+            return null;
+        }
+    }
+    function transformResponse(raw) {
+        const response = raw;
+        const content = response?.results?.[0]?.content;
+        if (!content)
+            return null;
+        const answerText = (0, scrape_js_1.cleanAnswer)(content.response_text || '');
+        const answerTextMarkdown = (0, scrape_js_1.cleanAnswer)(content.markdown_text || '');
+        // Map section='citations' to cited=true (like Brightdata's cited field)
+        const citations = (content.citations ?? []).map((c) => ({
+            ...c,
+            cited: c.section === 'citations',
+        }));
+        return {
+            prompt: content.prompt || '',
+            answer: answerText,
+            answer_text_markdown: answerTextMarkdown,
+            sources: (0, scrape_js_1.buildSources)(citations),
+            searchQueries: [],
+        };
+    }
+    return {
+        name: config.providerName,
+        maxConcurrency: config.maxConcurrency,
+        maxPromptsPerRequest: config.maxPromptsPerRequest,
+        triggerJob,
+        monitorJob,
+        downloadJob,
+        transformResponse,
+    };
+}
+// ============================================================================
+// Export
+// ============================================================================
+exports.oxylabsProvider = createOxylabsProvider();

package/{esm/src/apis/chatgptScraper/scraper.d.ts → script/src/apis/brightdata/llmScraper/scrape.d.ts}

@@ -1,5 +1,5 @@
-import type { ModelResult } from '../../schemas/models.schema.js';
-import type { Source, SearchSource } from '../../schemas/sources.schema.js';
+import type { ModelResult } from '../../../schemas/models.schema.js';
+import type { Source } from '../../../schemas/sources.schema.js';
 export interface BatchOptions {
     prompts: Array<string>;
     useSearch?: boolean;
@@ -14,12 +14,12 @@ export interface ProviderFunctions {
     downloadJob: (jobId: string) => Promise<unknown>;
     transformResponse: (raw: unknown) => ModelResult | null;
 }
-export interface GPTScraper {
+export interface LLMScraper {
     maxConcurrency: number;
     maxPromptsPerRequest: number;
-    scrapeGPTBatch: (options: BatchOptions) => Promise<Array<ModelResult>>;
-    triggerGPTBatch: (options: BatchOptions) => Promise<Array<string | null>>;
-    downloadGPTSnapshots: (jobIds: Array<string | null>) => Promise<Array<ModelResult>>;
+    scrapeLLMBatch: (options: BatchOptions) => Promise<Array<ModelResult>>;
+    triggerLLMBatch: (options: BatchOptions) => Promise<Array<string | null>>;
+    downloadLLMSnapshots: (jobIds: Array<string | null>) => Promise<Array<ModelResult>>;
 }
 export declare function getAbortSignal(): AbortSignal | undefined;
 export declare function cleanAnswer(answer: string): string;
@@ -29,18 +29,15 @@ export declare function buildSources(citations: Array<{
     description?: string;
     text?: string;
     cited?: boolean;
-}>, linkPositions?: Record<string, Array<number>>): Array<Source>;
-export declare function buildSearchSources(sources: Array<{
+}>, linksAttached?: Array<{
     url?: string;
-    title?: string;
-    snippet?: string;
-    rank?: number;
-    date_published?: string;
-}>): Array<SearchSource>;
+    text?: string;
+    position?: number;
+}>): Array<Source>;
 /**
  * Creates an empty model result for failed jobs.
  * This ensures we always return the same number of rows as input.
  */
 export declare function emptyModelResult(providerName: string, errorMessage?: string, context?: unknown): ModelResult;
-export declare function createScraper(provider: ProviderFunctions): GPTScraper;
-//# sourceMappingURL=scraper.d.ts.map
+export declare function createLLMScraper(provider: ProviderFunctions): LLMScraper;
+//# sourceMappingURL=scrape.d.ts.map

package/script/src/apis/brightdata/llmScraper/scrape.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scrape.d.ts","sourceRoot":"","sources":["../../../../../src/src/apis/brightdata/llmScraper/scrape.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mCAAmC,CAAC;AACrE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oCAAoC,CAAC;AAOjE,MAAM,WAAW,YAAY;IAC5B,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACvB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/B;AAED,MAAM,WAAW,iBAAiB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,EAAE,MAAM,CAAC;IACvB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,UAAU,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,GAAG,IAAI,KAAK,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAC1G,UAAU,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IAChD,WAAW,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IACjD,iBAAiB,EAAE,CAAC,GAAG,EAAE,OAAO,KAAK,WAAW,GAAG,IAAI,CAAC;CACxD;AAED,MAAM,WAAW,UAAU;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,cAAc,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC;IACvE,eAAe,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC;IAC1E,oBAAoB,EAAE,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC;CACpF;AAMD,wBAAgB,cAAc,IAAI,WAAW,GAAG,SAAS,CAExD;AAED,wBAAgB,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAMlD;AA+BD,wBAAgB,YAAY,CAC3B,SAAS,EAAE,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,OAAO,CAAA;CAAE,CAAC,EACvG,aAAa,GAAE,KAAK,CAAC;IAAE,GAAG,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,CAAM,GAC3E,KAAK,CAAC,MAAM,CAAC,CAoFf;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,YAAY,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,GAAG,WAAW,CAU5G;AAMD,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,iBAAiB,GAAG,UAAU,CAkExE"}

package/script/src/apis/brightdata/llmScraper/scrape.js

@@ -0,0 +1,224 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getAbortSignal = getAbortSignal;
+exports.cleanAnswer = cleanAnswer;
+exports.buildSources = buildSources;
+exports.emptyModelResult = emptyModelResult;
+exports.createLLMScraper = createLLMScraper;
+/* eslint no-console: ["warn", { allow: ["log", "warn", "error"] }] */
+/**
+ * LLM Scraper - Core types and orchestration logic.
+ *
+ * Uses composition: providers supply functions, this module orchestrates them.
+ */
+const dntShim = __importStar(require("../../../../_dnt.shims.js"));
+const async_js_1 = require("../../../helpers/async.js");
+const urls_js_1 = require("../../../helpers/urls.js");
+// ============================================================================
+// Shared Utilities
+// ============================================================================
+function getAbortSignal() {
+    return dntShim.dntGlobalThis.abortSignal;
+}
+function cleanAnswer(answer) {
+    return answer
+        .replace(/!\[([^\]]*)\]\([^)]+\)/g, '')
+        .replace(/\n\s*Image\s*\n/g, '\n')
+        .replace(/\n{3,}/g, '\n\n')
+        .trim();
+}
+/**
+ * Derive a merge key from a URL: origin + pathname, stripping query and fragment.
+ * Falls back to the raw URL if parsing fails.
+ */
+function urlMergeKey(url) {
+    try {
+        const parsed = new URL(url);
+        return parsed.origin + parsed.pathname;
+    }
+    catch {
+        return url;
+    }
+}
+/**
+ * Returns true when `candidate` carries extra info (hash or search params)
+ * that `current` does not.
+ */
+function hasExtraUrlInfo(current, candidate) {
+    try {
+        const cur = new URL(current);
+        const cand = new URL(candidate);
+        const hasNewHash = cand.hash !== '' && cur.hash === '';
+        const hasNewParams = cand.search !== '' && cur.search === '';
+        return hasNewHash || hasNewParams;
+    }
+    catch {
+        return false;
+    }
+}
+function buildSources(citations, linksAttached = []) {
+    const sources = [];
+    const sourcesByKey = new Map();
+    const upsertSource = (url, initialTitle, cited) => {
+        const key = urlMergeKey(url);
+        const existing = sourcesByKey.get(key);
+        if (existing) {
+            if (!existing.title && initialTitle) {
+                existing.title = initialTitle;
+            }
+            existing.cited = existing.cited || cited;
+            // Keep the most informative URL (with fragment/params)
+            if (hasExtraUrlInfo(existing.url, url)) {
+                existing.url = url;
+            }
+            return existing;
+        }
+        const source = {
+            title: initialTitle,
+            url,
+            domain: (0, urls_js_1.extractDomain)(url),
+            cited,
+        };
+        sources.push(source);
+        sourcesByKey.set(key, source);
+        return source;
+    };
+    const sortedLinks = [...linksAttached].sort((a, b) => {
+        const aPos = a.position ?? Number.MAX_SAFE_INTEGER;
+        const bPos = b.position ?? Number.MAX_SAFE_INTEGER;
+        return aPos - bPos;
+    });
+    for (const link of sortedLinks) {
+        if (!link.url)
+            continue;
+        const source = upsertSource(link.url, link.text ?? '', true);
+        if (link.position != null) {
+            source.positions ??= [];
+            if (!source.positions.includes(link.position)) {
+                source.positions.push(link.position);
+            }
+        }
+    }
+    for (const citation of citations) {
+        if (!citation.url)
+            continue;
+        const key = urlMergeKey(citation.url);
+        const existing = sourcesByKey.get(key);
+        const title = citation.title || citation.description || citation.text || '';
+        if (existing) {
+            if (title) {
+                existing.title = title;
+            }
+            existing.cited = existing.cited || citation.cited;
+            // Append extra fragment/params from citation
+            if (hasExtraUrlInfo(existing.url, citation.url)) {
+                existing.url = citation.url;
+            }
+            continue;
+        }
+        const source = {
+            title,
+            url: citation.url,
+            domain: (0, urls_js_1.extractDomain)(citation.url),
+            cited: citation.cited,
+        };
+        sources.push(source);
+        sourcesByKey.set(key, source);
+    }
+    for (const source of sources) {
+        source.positions?.sort((a, b) => a - b);
+    }
+    return sources;
+}
+/**
+ * Creates an empty model result for failed jobs.
+ * This ensures we always return the same number of rows as input.
+ */
+function emptyModelResult(providerName, errorMessage, context) {
+    if (errorMessage) {
+        console.error(`[${providerName}] ${errorMessage}`, context ?? '');
+    }
+    return {
+        prompt: '',
+        answer: '',
+        answer_text_markdown: '',
+        sources: [],
+    };
+}
+// ============================================================================
+// Scraper Factory
+// ============================================================================
+function createLLMScraper(provider) {
+    const { name, maxConcurrency, maxPromptsPerRequest, triggerJob, monitorJob, downloadJob, transformResponse, } = provider;
+    async function triggerLLMBatch({ prompts, useSearch = false, countryISOCode = null, }) {
+        const jobIds = await (0, async_js_1.mapParallel)(prompts, maxConcurrency, (prompt) => triggerJob(prompt, useSearch, countryISOCode));
+        console.log(`[${name}] Triggered ${jobIds.length} jobs for ${prompts.length} prompts`);
+        return jobIds;
+    }
+    async function downloadLLMSnapshots(jobIds) {
+        const results = [];
+        for (const jobId of jobIds) {
+            if (!jobId) {
+                results.push(emptyModelResult(name, 'No job ID provided'));
+                continue;
+            }
+            const isReady = await monitorJob(jobId);
+            if (!isReady) {
+                results.push(emptyModelResult(name, 'Job not ready or failed', jobId));
+                continue;
+            }
+            const raw = await downloadJob(jobId);
+            if (!raw) {
+                results.push(emptyModelResult(name, 'Failed to download job', jobId));
+                continue;
+            }
+            const result = transformResponse(raw);
+            results.push(result ?? emptyModelResult(name, 'Failed to transform response', jobId));
+        }
+        return results;
+    }
+    async function scrapeLLMBatch(options) {
+        const jobIds = await triggerLLMBatch(options);
+        return downloadLLMSnapshots(jobIds);
+    }
+    return {
+        maxConcurrency,
+        maxPromptsPerRequest,
+        scrapeLLMBatch,
+        triggerLLMBatch,
+        downloadLLMSnapshots,
+    };
+}

package/script/src/schemas/search.schema.d.ts

@@ -1,12 +1,12 @@
 import type { z } from '../../deps/jsr.io/@zod/zod/4.3.6/src/index.js';
-import type { Source, SearchSource } from './sources.schema.js';
+import type { Source } from './sources.schema.js';
 export type ContextSize = 'low' | 'medium' | 'high';
 export type ReasoningEffort = 'low' | 'medium' | 'high';
 export interface SearchResult {
     answer: string;
+    answer_text_markdown?: string;
     sources: Array<Source>;
     searchQueries?: Array<string>;
-    searchSources?: Array<SearchSource>;
 }
 export type SearchOptions = {
     prompt: string;

package/script/src/schemas/search.schema.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"search.schema.d.ts","sourceRoot":"","sources":["../../../src/src/schemas/search.schema.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,+CAA+C,CAAC;AAEvE,OAAO,KAAK,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAChE,MAAM,MAAM,WAAW,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;AACpD,MAAM,MAAM,eAAe,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;AAGxD,MAAM,WAAW,YAAY;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACvB,aAAa,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAC9B,aAAa,CAAC,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;CACpC;AAED,MAAM,MAAM,aAAa,GAAG;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;4FACwF;IACxF,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,UAAU,CAAC,EAAE,YAAY,GAAG,oBAAoB,CAAA;CAChD,CAAC;AAEF,MAAM,MAAM,sBAAsB,CAAC,CAAC,IAAI,aAAa,GAAG;IACvD,cAAc,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;CAC7B,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC,GAAG;IAChE,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAA;CACvB,CAAA"}
+{"version":3,"file":"search.schema.d.ts","sourceRoot":"","sources":["../../../src/src/schemas/search.schema.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,+CAA+C,CAAC;AAEvE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAClD,MAAM,MAAM,WAAW,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;AACpD,MAAM,MAAM,eAAe,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;AAExD,MAAM,WAAW,YAAY;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACvB,aAAa,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;CAC9B;AAED,MAAM,MAAM,aAAa,GAAG;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;4FACwF;IACxF,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,UAAU,CAAC,EAAE,YAAY,GAAG,oBAAoB,CAAC;CACjD,CAAC;AAEF,MAAM,MAAM,sBAAsB,CAAC,CAAC,IAAI,aAAa,GAAG;IACvD,cAAc,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;CAC7B,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC,GAAG;IAChE,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;CACxB,CAAC"}