@graphext/cuery 0.9.5 → 0.10.1

package/script/src/apis/brightdata/llmScraper/scrape.js

@@ -0,0 +1,224 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getAbortSignal = getAbortSignal;
+exports.cleanAnswer = cleanAnswer;
+exports.buildSources = buildSources;
+exports.emptyModelResult = emptyModelResult;
+exports.createLLMScraper = createLLMScraper;
+/* eslint no-console: ["warn", { allow: ["log", "warn", "error"] }] */
+/**
+ * LLM Scraper - Core types and orchestration logic.
+ *
+ * Uses composition: providers supply functions, this module orchestrates them.
+ */
+const dntShim = __importStar(require("../../../../_dnt.shims.js"));
+const async_js_1 = require("../../../helpers/async.js");
+const urls_js_1 = require("../../../helpers/urls.js");
+// ============================================================================
+// Shared Utilities
+// ============================================================================
+function getAbortSignal() {
+    return dntShim.dntGlobalThis.abortSignal;
+}
+function cleanAnswer(answer) {
+    return answer
+        .replace(/!\[([^\]]*)\]\([^)]+\)/g, '')
+        .replace(/\n\s*Image\s*\n/g, '\n')
+        .replace(/\n{3,}/g, '\n\n')
+        .trim();
+}
+/**
+ * Derive a merge key from a URL: origin + pathname, stripping query and fragment.
+ * Falls back to the raw URL if parsing fails.
+ */
+function urlMergeKey(url) {
+    try {
+        const parsed = new URL(url);
+        return parsed.origin + parsed.pathname;
+    }
+    catch {
+        return url;
+    }
+}
+/**
+ * Returns true when `candidate` carries extra info (hash or search params)
+ * that `current` does not.
+ */
+function hasExtraUrlInfo(current, candidate) {
+    try {
+        const cur = new URL(current);
+        const cand = new URL(candidate);
+        const hasNewHash = cand.hash !== '' && cur.hash === '';
+        const hasNewParams = cand.search !== '' && cur.search === '';
+        return hasNewHash || hasNewParams;
+    }
+    catch {
+        return false;
+    }
+}
+function buildSources(citations, linksAttached = []) {
+    const sources = [];
+    const sourcesByKey = new Map();
+    const upsertSource = (url, initialTitle, cited) => {
+        const key = urlMergeKey(url);
+        const existing = sourcesByKey.get(key);
+        if (existing) {
+            if (!existing.title && initialTitle) {
+                existing.title = initialTitle;
+            }
+            existing.cited = existing.cited || cited;
+            // Keep the most informative URL (with fragment/params)
+            if (hasExtraUrlInfo(existing.url, url)) {
+                existing.url = url;
+            }
+            return existing;
+        }
+        const source = {
+            title: initialTitle,
+            url,
+            domain: (0, urls_js_1.extractDomain)(url),
+            cited,
+        };
+        sources.push(source);
+        sourcesByKey.set(key, source);
+        return source;
+    };
+    const sortedLinks = [...linksAttached].sort((a, b) => {
+        const aPos = a.position ?? Number.MAX_SAFE_INTEGER;
+        const bPos = b.position ?? Number.MAX_SAFE_INTEGER;
+        return aPos - bPos;
+    });
+    for (const link of sortedLinks) {
+        if (!link.url)
+            continue;
+        const source = upsertSource(link.url, link.text ?? '', true);
+        if (link.position != null) {
+            source.positions ??= [];
+            if (!source.positions.includes(link.position)) {
+                source.positions.push(link.position);
+            }
+        }
+    }
+    for (const citation of citations) {
+        if (!citation.url)
+            continue;
+        const key = urlMergeKey(citation.url);
+        const existing = sourcesByKey.get(key);
+        const title = citation.title || citation.description || citation.text || '';
+        if (existing) {
+            if (title) {
+                existing.title = title;
+            }
+            existing.cited = existing.cited || citation.cited;
+            // Append extra fragment/params from citation
+            if (hasExtraUrlInfo(existing.url, citation.url)) {
+                existing.url = citation.url;
+            }
+            continue;
+        }
+        const source = {
+            title,
+            url: citation.url,
+            domain: (0, urls_js_1.extractDomain)(citation.url),
+            cited: citation.cited,
+        };
+        sources.push(source);
+        sourcesByKey.set(key, source);
+    }
+    for (const source of sources) {
+        source.positions?.sort((a, b) => a - b);
+    }
+    return sources;
+}
+/**
+ * Creates an empty model result for failed jobs.
+ * This ensures we always return the same number of rows as input.
+ */
+function emptyModelResult(providerName, errorMessage, context) {
+    if (errorMessage) {
+        console.error(`[${providerName}] ${errorMessage}`, context ?? '');
+    }
+    return {
+        prompt: '',
+        answer: '',
+        answer_text_markdown: '',
+        sources: [],
+    };
+}
+// ============================================================================
+// Scraper Factory
+// ============================================================================
+function createLLMScraper(provider) {
+    const { name, maxConcurrency, maxPromptsPerRequest, triggerJob, monitorJob, downloadJob, transformResponse, } = provider;
+    async function triggerLLMBatch({ prompts, useSearch = false, countryISOCode = null, }) {
+        const jobIds = await (0, async_js_1.mapParallel)(prompts, maxConcurrency, (prompt) => triggerJob(prompt, useSearch, countryISOCode));
+        console.log(`[${name}] Triggered ${jobIds.length} jobs for ${prompts.length} prompts`);
+        return jobIds;
+    }
+    async function downloadLLMSnapshots(jobIds) {
+        const results = [];
+        for (const jobId of jobIds) {
+            if (!jobId) {
+                results.push(emptyModelResult(name, 'No job ID provided'));
+                continue;
+            }
+            const isReady = await monitorJob(jobId);
+            if (!isReady) {
+                results.push(emptyModelResult(name, 'Job not ready or failed', jobId));
+                continue;
+            }
+            const raw = await downloadJob(jobId);
+            if (!raw) {
+                results.push(emptyModelResult(name, 'Failed to download job', jobId));
+                continue;
+            }
+            const result = transformResponse(raw);
+            results.push(result ?? emptyModelResult(name, 'Failed to transform response', jobId));
+        }
+        return results;
+    }
+    async function scrapeLLMBatch(options) {
+        const jobIds = await triggerLLMBatch(options);
+        return downloadLLMSnapshots(jobIds);
+    }
+    return {
+        maxConcurrency,
+        maxPromptsPerRequest,
+        scrapeLLMBatch,
+        triggerLLMBatch,
+        downloadLLMSnapshots,
+    };
+}

package/script/src/apis/hasdata/helpers.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"helpers.d.ts","sourceRoot":"","sources":["../../../../src/src/apis/hasdata/helpers.ts"],"names":[],"mappings":"AACA,OAAO,EAAe,KAAK,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAEvE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,iCAAiC,CAAC;AAG9D,eAAO,MAAM,mBAAmB,KAAK,CAAC;AAEtC,eAAO,MAAM,oBAAoB,EAAE,WAMlC,CAAC;AAEF,wBAAgB,gBAAgB,IAAI,MAAM,CAMzC;AAED,wBAAsB,qBAAqB,CAC1C,GAAG,EAAE,MAAM,EACX,WAAW,GAAE,WAAkC,GAC7C,OAAO,CAAC,QAAQ,CAAC,CAgCnB;AAED,UAAU,QAAQ;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;CACvB;AAED,UAAU,SAAS;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,uBAAuB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACxC,gBAAgB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACjC,IAAI,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;IACvB,IAAI,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IAC5B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,UAAU,SAAS;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,UAAU;IAC1B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,UAAU,eAAe;IACxB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,MAAM;IACtB,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;CAC9B;AAED,MAAM,WAAW,SAAS;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;CACvB;AAwJD,wBAAgB,QAAQ,CAAC,GAAG,EAAE,UAAU,GAAG,SAAS,CAInD;AAED,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAI/C"}
+{"version":3,"file":"helpers.d.ts","sourceRoot":"","sources":["../../../../src/src/apis/hasdata/helpers.ts"],"names":[],"mappings":"AACA,OAAO,EAAe,KAAK,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAEvE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,iCAAiC,CAAC;AAG9D,eAAO,MAAM,mBAAmB,KAAK,CAAC;AAEtC,eAAO,MAAM,oBAAoB,EAAE,WAMlC,CAAC;AAEF,wBAAgB,gBAAgB,IAAI,MAAM,CAMzC;AAED,wBAAsB,qBAAqB,CAC1C,GAAG,EAAE,MAAM,EACX,WAAW,GAAE,WAAkC,GAC7C,OAAO,CAAC,QAAQ,CAAC,CAgCnB;AAED,UAAU,QAAQ;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;CACvB;AAED,UAAU,SAAS;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,uBAAuB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACxC,gBAAgB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACjC,IAAI,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;IACvB,IAAI,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IAC5B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,UAAU,SAAS;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,UAAU;IAC1B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,UAAU,eAAe;IACxB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,MAAM;IACtB,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;CAC9B;AAED,MAAM,WAAW,SAAS;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;CACvB;AAqMD,wBAAgB,QAAQ,CAAC,GAAG,EAAE,UAAU,GAAG,SAAS,CAInD;AAED,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAI/C"}

package/script/src/apis/hasdata/helpers.js

@@ -156,8 +156,38 @@ function formatCode(block) {
     const header = `[Code${lang ? ': ' + lang : ''}]`;
     return `${header}\n${snippet.trim()}`;
 }
+function formatCitationMarkers(refIndexes) {
+    if (refIndexes.length === 0) {
+        return '';
+    }
+    return ' ' + refIndexes.map(i => `[${i + 1}]`).join('');
+}
 function parseAIResult(data, { allowNestedOverview = true } = {}) {
     const textBlocks = data.textBlocks || (allowNestedOverview ? data.aiOverview?.textBlocks : []) || [];
+    // Build reference index → source index mapping and track cited refs
+    const refs = data.references || (allowNestedOverview ? data.aiOverview?.references : []) || [];
+    const sources = [];
+    const refIndexToSourceIndex = new Map();
+    for (const r of refs) {
+        const link = r.link || r.url;
+        const title = [r.title, r.source, r.snippet].filter(Boolean).join(' - ');
+        if (link && r.index != null) {
+            // Deduplicate by URL
+            const existingIdx = sources.findIndex(s => s.url === link);
+            if (existingIdx >= 0) {
+                refIndexToSourceIndex.set(r.index, existingIdx);
+            }
+            else {
+                refIndexToSourceIndex.set(r.index, sources.length);
+                sources.push({
+                    title,
+                    url: link,
+                    domain: (0, urls_js_1.extractDomain)(link)
+                });
+            }
+        }
+    }
+    const citedSourceIndexes = new Set();
     const parts = [];
     const handlers = {
         paragraph: (b) => cleanText(b.snippet || ''),
@@ -171,19 +201,40 @@ function parseAIResult(data, { allowNestedOverview = true } = {}) {
             continue;
         }
         const handler = handlers[btype];
+        let rendered = '';
         if (handler) {
-            const rendered = handler(block);
-            if (rendered) {
-                parts.push(rendered);
-            }
+            rendered = handler(block);
         }
         else {
             const snippet = block.snippet || '';
             if (snippet) {
-                parts.push(cleanText(snippet));
+                rendered = cleanText(snippet);
+            }
+        }
+        if (rendered) {
+            // Append citation markers and track positions
+            const refIndexes = block.referenceIndexes || [];
+            if (refIndexes.length > 0) {
+                // Map ref indexes to 1-based source indexes for display
+                const sourceIndexes = refIndexes
+                    .map(ri => refIndexToSourceIndex.get(ri))
+                    .filter((si) => si != null);
+                for (const si of sourceIndexes) {
+                    citedSourceIndexes.add(si);
+                    sources[si].positions ??= [];
+                    if (!sources[si].positions.includes(parts.length)) {
+                        sources[si].positions.push(parts.length);
+                    }
+                }
+                rendered += formatCitationMarkers(sourceIndexes.filter((v, i, a) => a.indexOf(v) === i));
             }
+            parts.push(rendered);
         }
     }
+    // Mark cited sources
+    for (const si of citedSourceIndexes) {
+        sources[si].cited = true;
+    }
     const deduped = [];
     for (const p of parts) {
         if (deduped.length === 0 || deduped[deduped.length - 1] !== p) {
@@ -195,19 +246,6 @@ function parseAIResult(data, { allowNestedOverview = true } = {}) {
         console.warn('Warning: AI answer truncated to 16000 characters');
         answer = answer.slice(0, 16000);
     }
-    const refs = data.references || (allowNestedOverview ? data.aiOverview?.references : []) || [];
-    const sources = [];
-    for (const r of refs) {
-        const link = r.link || r.url;
-        const title = [r.title, r.source, r.snippet].filter(Boolean).join(' - ');
-        if (link) {
-            sources.push({
-                title,
-                url: link,
-                domain: (0, urls_js_1.extractDomain)(link)
-            });
-        }
-    }
     return { answer, sources };
 }
 function parseAIO(aio) {

package/script/src/schemas/search.schema.d.ts

@@ -1,12 +1,12 @@
 import type { z } from '../../deps/jsr.io/@zod/zod/4.3.6/src/index.js';
-import type { Source, SearchSource } from './sources.schema.js';
+import type { Source } from './sources.schema.js';
 export type ContextSize = 'low' | 'medium' | 'high';
 export type ReasoningEffort = 'low' | 'medium' | 'high';
 export interface SearchResult {
     answer: string;
+    answer_text_markdown?: string;
     sources: Array<Source>;
     searchQueries?: Array<string>;
-    searchSources?: Array<SearchSource>;
 }
 export type SearchOptions = {
     prompt: string;

package/script/src/schemas/search.schema.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"search.schema.d.ts","sourceRoot":"","sources":["../../../src/src/schemas/search.schema.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,+CAA+C,CAAC;AAEvE,OAAO,KAAK,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAChE,MAAM,MAAM,WAAW,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;AACpD,MAAM,MAAM,eAAe,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;AAGxD,MAAM,WAAW,YAAY;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACvB,aAAa,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAC9B,aAAa,CAAC,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;CACpC;AAED,MAAM,MAAM,aAAa,GAAG;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;4FACwF;IACxF,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,UAAU,CAAC,EAAE,YAAY,GAAG,oBAAoB,CAAA;CAChD,CAAC;AAEF,MAAM,MAAM,sBAAsB,CAAC,CAAC,IAAI,aAAa,GAAG;IACvD,cAAc,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;CAC7B,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC,GAAG;IAChE,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAA;CACvB,CAAA"}
+{"version":3,"file":"search.schema.d.ts","sourceRoot":"","sources":["../../../src/src/schemas/search.schema.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,+CAA+C,CAAC;AAEvE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAClD,MAAM,MAAM,WAAW,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;AACpD,MAAM,MAAM,eAAe,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;AAExD,MAAM,WAAW,YAAY;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACvB,aAAa,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;CAC9B;AAED,MAAM,MAAM,aAAa,GAAG;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;4FACwF;IACxF,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,UAAU,CAAC,EAAE,YAAY,GAAG,oBAAoB,CAAC;CACjD,CAAC;AAEF,MAAM,MAAM,sBAAsB,CAAC,CAAC,IAAI,aAAa,GAAG;IACvD,cAAc,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;CAC7B,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC,GAAG;IAChE,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;CACxB,CAAC"}

package/script/src/schemas/sources.schema.d.ts

@@ -3,6 +3,7 @@ export interface Source {
     url: string;
     domain: string;
     cited?: boolean;
+    snippet?: string;
     positions?: Array<number>;
 }
 /**
@@ -22,8 +23,4 @@ export interface CategorizedSource extends EnrichedSource {
     category: string | null;
     subcategory: string | null;
 }
-export interface SearchSource extends Source {
-    rank: number;
-    datePublished: string | null;
-}
 //# sourceMappingURL=sources.schema.d.ts.map

package/script/src/schemas/sources.schema.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"sources.schema.d.ts","sourceRoot":"","sources":["../../../src/src/schemas/sources.schema.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,MAAM;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,SAAS,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;CAC1B;AAED;;;GAGG;AACH,MAAM,WAAW,cAAe,SAAQ,MAAM;IAC7C,eAAe,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAC/B,oBAAoB,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACpC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,iBAAkB,SAAQ,cAAc;IACxD,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3B;AAED,MAAM,WAAW,YAAa,SAAQ,MAAM;IAC3C,IAAI,EAAE,MAAM,CAAC;IACb,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B"}
+{"version":3,"file":"sources.schema.d.ts","sourceRoot":"","sources":["../../../src/src/schemas/sources.schema.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,MAAM;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;CAC1B;AAED;;;GAGG;AACH,MAAM,WAAW,cAAe,SAAQ,MAAM;IAC7C,eAAe,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAC/B,oBAAoB,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACpC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,iBAAkB,SAAQ,cAAc;IACxD,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3B"}

package/esm/src/apis/brightdata/index.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/src/apis/brightdata/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC"}

package/esm/src/apis/brightdata/scrape.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"scrape.d.ts","sourceRoot":"","sources":["../../../../src/src/apis/brightdata/scrape.ts"],"names":[],"mappings":"AACA,OAAO,EAA4B,KAAK,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAapF,MAAM,WAAW,uBAAuB;IACpC,wDAAwD;IACxD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,6DAA6D;IAC7D,MAAM,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;IACxB,8DAA8D;IAC9D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,wDAAwD;IACxD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oDAAoD;IACpD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,kCAAkC;IAClC,WAAW,CAAC,EAAE,WAAW,CAAC;CAC7B;AAED,MAAM,WAAW,wBAAwB;IACrC,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;CACjB;AA4ED;;;GAGG;AACH,wBAAsB,gBAAgB,CAClC,GAAG,EAAE,MAAM,EACX,OAAO,GAAE,uBAA4B,GACtC,OAAO,CAAC,wBAAwB,CAAC,CAYnC;AAED;;;GAGG;AACH,wBAAsB,qBAAqB,CACvC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,EACnB,OAAO,GAAE,uBAA4B,EACrC,cAAc,GAAE,MAA+B,GAChD,OAAO,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC,CAQ1C"}

package/esm/src/apis/chatgptScraper/brightdata.d.ts

@@ -1,3 +0,0 @@
-import { type ProviderFunctions } from './scraper.js';
-export declare const brightdataProvider: ProviderFunctions;
-//# sourceMappingURL=brightdata.d.ts.map

package/esm/src/apis/chatgptScraper/brightdata.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"brightdata.d.ts","sourceRoot":"","sources":["../../../../src/src/apis/chatgptScraper/brightdata.ts"],"names":[],"mappings":"AAeA,OAAO,EACN,KAAK,iBAAiB,EAKtB,MAAM,cAAc,CAAC;AAmNtB,eAAO,MAAM,kBAAkB,EAAE,iBAQhC,CAAC"}

package/esm/src/apis/chatgptScraper/brightdata.js

@@ -1,172 +0,0 @@
-/* eslint no-console: ["warn", { allow: ["log", "warn", "error"] }] */
-/**
- * Brightdata GPT Scraper Provider.
- *
- * API Flow:
- * 1. Trigger: POST to /datasets/v3/trigger → returns snapshot_id
- * 2. Monitor: GET /datasets/v3/progress/{snapshot_id} until ready
- * 3. Download: GET /datasets/v3/snapshot/{snapshot_id}
- */
-import * as dntShim from "../../../_dnt.shims.js";
-import { withRetries, sleep } from '../../helpers/async.js';
-import { getAbortSignal, cleanAnswer, buildSources, buildSearchSources } from './scraper.js';
-// ============================================================================
-// Constants
-// ============================================================================
-const API_BASE = 'https://api.brightdata.com';
-const DATASET_ID = 'gd_m7aof0k82r803d5bjm';
-const OUTPUT_FIELDS = 'url|prompt|answer_text|answer_text_markdown|citations|links_attached|search_sources|country|model|web_search_triggered|web_search_query|index';
-const TRIGGER_RETRY = {
-    maxRetries: 3,
-    initialDelay: 0,
-    statusCodes: [429, 500, 502, 503, 504]
-};
-const DOWNLOAD_RETRY = {
-    maxRetries: 5,
-    initialDelay: 2000,
-    statusCodes: [202, 500, 502, 503, 504]
-};
-const MONITOR_RETRY = {
-    maxRetries: 4,
-    initialDelay: 1000,
-    statusCodes: [408, 425, 429, 500, 502, 503, 504]
-};
-const MONITOR_RETRIABLE = new Set(MONITOR_RETRY.statusCodes ?? []);
-const MAX_WAIT_MS = 600_000; // 10 minutes
-const POLL_INTERVAL_MS = 5_000;
-// ============================================================================
-// API Key
-// ============================================================================
-function getApiKey() {
-    const apiKey = dntShim.Deno.env.get('BRIGHTDATA_API_KEY');
-    if (!apiKey) {
-        throw new Error('BRIGHTDATA_API_KEY environment variable is required');
-    }
-    return apiKey;
-}
-// ============================================================================
-// Provider Functions
-// ============================================================================
-async function triggerJob(prompt, useSearch, countryISOCode) {
-    const apiKey = getApiKey();
-    const url = `${API_BASE}/datasets/v3/trigger?dataset_id=${DATASET_ID}&include_errors=true`;
-    const body = {
-        custom_output_fields: OUTPUT_FIELDS,
-        input: [{
-                url: 'http://chatgpt.com/',
-                prompt,
-                web_search: useSearch,
-                country: countryISOCode || '',
-                index: 0
-            }]
-    };
-    try {
-        const response = await withRetries(() => fetch(url, {
-            method: 'POST',
-            headers: {
-                'Authorization': `Bearer ${apiKey}`,
-                'Content-Type': 'application/json'
-            },
-            body: JSON.stringify(body),
-            signal: getAbortSignal()
-        }), TRIGGER_RETRY);
-        if (!response.ok) {
-            console.error(`[Brightdata] Trigger error: ${response.status}`);
-            return null;
-        }
-        const data = await response.json();
-        return data?.snapshot_id || null;
-    }
-    catch (error) {
-        console.error('[Brightdata] Trigger failed:', error);
-        return null;
-    }
-}
-async function monitorJob(snapshotId) {
-    const apiKey = getApiKey();
-    const url = `${API_BASE}/datasets/v3/progress/${snapshotId}`;
-    const startTime = Date.now();
-    const abortSignal = getAbortSignal();
-    while (Date.now() - startTime < MAX_WAIT_MS) {
-        if (abortSignal?.aborted)
-            return false;
-        try {
-            const response = await withRetries(() => fetch(url, {
-                headers: { 'Authorization': `Bearer ${apiKey}` },
-                signal: abortSignal
-            }), MONITOR_RETRY);
-            if (!response.ok) {
-                if (!MONITOR_RETRIABLE.has(response.status))
-                    return false;
-            }
-            else {
-                const status = await response.json();
-                if (status.status === 'ready' || status.status === 'complete')
-                    return true;
-                if (status.status === 'failed' || status.status === 'error')
-                    return false;
-            }
-        }
-        catch (error) {
-            console.error('[Brightdata] Monitor error:', error);
-        }
-        await sleep(POLL_INTERVAL_MS, abortSignal);
-    }
-    console.error(`[Brightdata] Monitor timeout after ${MAX_WAIT_MS / 1000}s`);
-    return false;
-}
-async function downloadJob(snapshotId) {
-    const apiKey = getApiKey();
-    const url = `${API_BASE}/datasets/v3/snapshot/${snapshotId}?format=json`;
-    try {
-        const response = await withRetries(() => fetch(url, {
-            headers: { 'Authorization': `Bearer ${apiKey}` },
-            signal: getAbortSignal()
-        }), DOWNLOAD_RETRY);
-        if (!response.ok) {
-            console.error(`[Brightdata] Download error: ${response.status}`);
-            return null;
-        }
-        const data = await response.json();
-        return Array.isArray(data) ? data : null;
-    }
-    catch (error) {
-        console.error('[Brightdata] Download failed:', error);
-        return null;
-    }
-}
-function transformResponse(raw) {
-    const responses = raw;
-    if (!responses || responses.length === 0)
-        return null;
-    const response = responses[0];
-    let answer = response.answer_text_markdown || response.answer_text || '';
-    answer = cleanAnswer(answer);
-    // Build link positions map
-    const linkPositions = {};
-    for (const link of response.links_attached ?? []) {
-        if (link.url && link.position != null) {
-            linkPositions[link.url] ??= [];
-            linkPositions[link.url].push(link.position);
-        }
-    }
-    return {
-        prompt: response.prompt,
-        answer,
-        sources: buildSources(response.citations ?? [], linkPositions),
-        searchQueries: response.web_search_query || [],
-        searchSources: buildSearchSources(response.search_sources ?? [])
-    };
-}
-// ============================================================================
-// Export
-// ============================================================================
-export const brightdataProvider = {
-    name: 'Brightdata',
-    maxConcurrency: 50,
-    maxPromptsPerRequest: 1,
-    triggerJob,
-    monitorJob,
-    downloadJob,
-    transformResponse
-};

package/esm/src/apis/chatgptScraper/index.d.ts

@@ -1,10 +0,0 @@
-import type { ModelResult } from '../../schemas/models.schema.js';
-import { type BatchOptions } from './scraper.js';
-export type { BatchOptions };
-export type JobId = string | null;
-export declare function getMaxConcurrency(): number;
-export declare function getMaxPromptsPerRequest(): number;
-export declare function scrapeGPTBatch(options: BatchOptions): Promise<Array<ModelResult>>;
-export declare function triggerGPTBatch(options: BatchOptions): Promise<Array<string | null>>;
-export declare function downloadGPTSnapshots(jobIds: Array<string | null>): Promise<Array<ModelResult>>;
-//# sourceMappingURL=index.d.ts.map

package/esm/src/apis/chatgptScraper/index.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/src/apis/chatgptScraper/index.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,gCAAgC,CAAC;AAClE,OAAO,EAAE,KAAK,YAAY,EAAkC,MAAM,cAAc,CAAC;AAKjF,YAAY,EAAE,YAAY,EAAE,CAAC;AAC7B,MAAM,MAAM,KAAK,GAAG,MAAM,GAAG,IAAI,CAAC;AAqBlC,wBAAgB,iBAAiB,IAAI,MAAM,CAE1C;AAED,wBAAgB,uBAAuB,IAAI,MAAM,CAEhD;AAED,wBAAsB,cAAc,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAEvF;AAED,wBAAsB,eAAe,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAE1F;AAED,wBAAsB,oBAAoB,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAEpG"}

package/esm/src/apis/chatgptScraper/index.js

@@ -1,41 +0,0 @@
-/* eslint no-console: ["warn", { allow: ["log", "warn", "error"] }] */
-/**
- * GPT Scraper - Public API
- *
- * Selects between Brightdata and Oxylabs based on CHATGPT_SCRAPER_PROVIDER env var.
- * Default: oxylabs
- */
-import * as dntShim from "../../../_dnt.shims.js";
-import { createScraper } from './scraper.js';
-import { brightdataProvider } from './brightdata.js';
-import { oxylabsProvider } from './oxy.js';
-// ============================================================================
-// Scraper Instance (lazy singleton)
-// ============================================================================
-let scraper = null;
-function getScraper() {
-    if (!scraper) {
-        const providerName = dntShim.Deno.env.get('CHATGPT_SCRAPER_PROVIDER')?.toLowerCase();
-        const provider = providerName === 'brightdata' ? brightdataProvider : oxylabsProvider;
-        scraper = createScraper(provider);
-    }
-    return scraper;
-}
-// ============================================================================
-// Public API
-// ============================================================================
-export function getMaxConcurrency() {
-    return getScraper().maxConcurrency;
-}
-export function getMaxPromptsPerRequest() {
-    return getScraper().maxPromptsPerRequest;
-}
-export async function scrapeGPTBatch(options) {
-    return getScraper().scrapeGPTBatch(options);
-}
-export async function triggerGPTBatch(options) {
-    return getScraper().triggerGPTBatch(options);
-}
-export async function downloadGPTSnapshots(jobIds) {
-    return getScraper().downloadGPTSnapshots(jobIds);
-}

package/esm/src/apis/chatgptScraper/oxy.d.ts

@@ -1,3 +0,0 @@
-import { type ProviderFunctions } from './scraper.js';
-export declare const oxylabsProvider: ProviderFunctions;
-//# sourceMappingURL=oxy.d.ts.map

package/esm/src/apis/chatgptScraper/oxy.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"oxy.d.ts","sourceRoot":"","sources":["../../../../src/src/apis/chatgptScraper/oxy.ts"],"names":[],"mappings":"AAeA,OAAO,EACN,KAAK,iBAAiB,EAItB,MAAM,cAAc,CAAC;AA6LtB,eAAO,MAAM,eAAe,EAAE,iBAQ7B,CAAC"}