@graphext/cuery 0.9.5 → 0.10.1

package/esm/src/apis/brightdata/llmScraper/scrape.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scrape.d.ts","sourceRoot":"","sources":["../../../../../src/src/apis/brightdata/llmScraper/scrape.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mCAAmC,CAAC;AACrE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oCAAoC,CAAC;AAOjE,MAAM,WAAW,YAAY;IAC5B,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACvB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/B;AAED,MAAM,WAAW,iBAAiB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,EAAE,MAAM,CAAC;IACvB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,UAAU,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,GAAG,IAAI,KAAK,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAC1G,UAAU,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IAChD,WAAW,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IACjD,iBAAiB,EAAE,CAAC,GAAG,EAAE,OAAO,KAAK,WAAW,GAAG,IAAI,CAAC;CACxD;AAED,MAAM,WAAW,UAAU;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,cAAc,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC;IACvE,eAAe,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC;IAC1E,oBAAoB,EAAE,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC;CACpF;AAMD,wBAAgB,cAAc,IAAI,WAAW,GAAG,SAAS,CAExD;AAED,wBAAgB,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAMlD;AA+BD,wBAAgB,YAAY,CAC3B,SAAS,EAAE,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,OAAO,CAAA;CAAE,CAAC,EACvG,aAAa,GAAE,KAAK,CAAC;IAAE,GAAG,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,CAAM,GAC3E,KAAK,CAAC,MAAM,CAAC,CAoFf;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,YAAY,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,GAAG,WAAW,CAU5G;AAMD,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,iBAAiB,GAAG,UAAU,CAkExE"}

package/esm/src/apis/brightdata/llmScraper/scrape.js

@@ -0,0 +1,184 @@
+/* eslint no-console: ["warn", { allow: ["log", "warn", "error"] }] */
+/**
+ * LLM Scraper - Core types and orchestration logic.
+ *
+ * Uses composition: providers supply functions, this module orchestrates them.
+ */
+import * as dntShim from "../../../../_dnt.shims.js";
+import { mapParallel } from '../../../helpers/async.js';
+import { extractDomain } from '../../../helpers/urls.js';
+// ============================================================================
+// Shared Utilities
+// ============================================================================
+export function getAbortSignal() {
+    return dntShim.dntGlobalThis.abortSignal;
+}
+export function cleanAnswer(answer) {
+    return answer
+        .replace(/!\[([^\]]*)\]\([^)]+\)/g, '')
+        .replace(/\n\s*Image\s*\n/g, '\n')
+        .replace(/\n{3,}/g, '\n\n')
+        .trim();
+}
+/**
+ * Derive a merge key from a URL: origin + pathname, stripping query and fragment.
+ * Falls back to the raw URL if parsing fails.
+ */
+function urlMergeKey(url) {
+    try {
+        const parsed = new URL(url);
+        return parsed.origin + parsed.pathname;
+    }
+    catch {
+        return url;
+    }
+}
+/**
+ * Returns true when `candidate` carries extra info (hash or search params)
+ * that `current` does not.
+ */
+function hasExtraUrlInfo(current, candidate) {
+    try {
+        const cur = new URL(current);
+        const cand = new URL(candidate);
+        const hasNewHash = cand.hash !== '' && cur.hash === '';
+        const hasNewParams = cand.search !== '' && cur.search === '';
+        return hasNewHash || hasNewParams;
+    }
+    catch {
+        return false;
+    }
+}
+export function buildSources(citations, linksAttached = []) {
+    const sources = [];
+    const sourcesByKey = new Map();
+    const upsertSource = (url, initialTitle, cited) => {
+        const key = urlMergeKey(url);
+        const existing = sourcesByKey.get(key);
+        if (existing) {
+            if (!existing.title && initialTitle) {
+                existing.title = initialTitle;
+            }
+            existing.cited = existing.cited || cited;
+            // Keep the most informative URL (with fragment/params)
+            if (hasExtraUrlInfo(existing.url, url)) {
+                existing.url = url;
+            }
+            return existing;
+        }
+        const source = {
+            title: initialTitle,
+            url,
+            domain: extractDomain(url),
+            cited,
+        };
+        sources.push(source);
+        sourcesByKey.set(key, source);
+        return source;
+    };
+    const sortedLinks = [...linksAttached].sort((a, b) => {
+        const aPos = a.position ?? Number.MAX_SAFE_INTEGER;
+        const bPos = b.position ?? Number.MAX_SAFE_INTEGER;
+        return aPos - bPos;
+    });
+    for (const link of sortedLinks) {
+        if (!link.url)
+            continue;
+        const source = upsertSource(link.url, link.text ?? '', true);
+        if (link.position != null) {
+            source.positions ??= [];
+            if (!source.positions.includes(link.position)) {
+                source.positions.push(link.position);
+            }
+        }
+    }
+    for (const citation of citations) {
+        if (!citation.url)
+            continue;
+        const key = urlMergeKey(citation.url);
+        const existing = sourcesByKey.get(key);
+        const title = citation.title || citation.description || citation.text || '';
+        if (existing) {
+            if (title) {
+                existing.title = title;
+            }
+            existing.cited = existing.cited || citation.cited;
+            // Append extra fragment/params from citation
+            if (hasExtraUrlInfo(existing.url, citation.url)) {
+                existing.url = citation.url;
+            }
+            continue;
+        }
+        const source = {
+            title,
+            url: citation.url,
+            domain: extractDomain(citation.url),
+            cited: citation.cited,
+        };
+        sources.push(source);
+        sourcesByKey.set(key, source);
+    }
+    for (const source of sources) {
+        source.positions?.sort((a, b) => a - b);
+    }
+    return sources;
+}
+/**
+ * Creates an empty model result for failed jobs.
+ * This ensures we always return the same number of rows as input.
+ */
+export function emptyModelResult(providerName, errorMessage, context) {
+    if (errorMessage) {
+        console.error(`[${providerName}] ${errorMessage}`, context ?? '');
+    }
+    return {
+        prompt: '',
+        answer: '',
+        answer_text_markdown: '',
+        sources: [],
+    };
+}
+// ============================================================================
+// Scraper Factory
+// ============================================================================
+export function createLLMScraper(provider) {
+    const { name, maxConcurrency, maxPromptsPerRequest, triggerJob, monitorJob, downloadJob, transformResponse, } = provider;
+    async function triggerLLMBatch({ prompts, useSearch = false, countryISOCode = null, }) {
+        const jobIds = await mapParallel(prompts, maxConcurrency, (prompt) => triggerJob(prompt, useSearch, countryISOCode));
+        console.log(`[${name}] Triggered ${jobIds.length} jobs for ${prompts.length} prompts`);
+        return jobIds;
+    }
+    async function downloadLLMSnapshots(jobIds) {
+        const results = [];
+        for (const jobId of jobIds) {
+            if (!jobId) {
+                results.push(emptyModelResult(name, 'No job ID provided'));
+                continue;
+            }
+            const isReady = await monitorJob(jobId);
+            if (!isReady) {
+                results.push(emptyModelResult(name, 'Job not ready or failed', jobId));
+                continue;
+            }
+            const raw = await downloadJob(jobId);
+            if (!raw) {
+                results.push(emptyModelResult(name, 'Failed to download job', jobId));
+                continue;
+            }
+            const result = transformResponse(raw);
+            results.push(result ?? emptyModelResult(name, 'Failed to transform response', jobId));
+        }
+        return results;
+    }
+    async function scrapeLLMBatch(options) {
+        const jobIds = await triggerLLMBatch(options);
+        return downloadLLMSnapshots(jobIds);
+    }
+    return {
+        maxConcurrency,
+        maxPromptsPerRequest,
+        scrapeLLMBatch,
+        triggerLLMBatch,
+        downloadLLMSnapshots,
+    };
+}

package/esm/src/apis/hasdata/helpers.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"helpers.d.ts","sourceRoot":"","sources":["../../../../src/src/apis/hasdata/helpers.ts"],"names":[],"mappings":"AACA,OAAO,EAAe,KAAK,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAEvE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,iCAAiC,CAAC;AAG9D,eAAO,MAAM,mBAAmB,KAAK,CAAC;AAEtC,eAAO,MAAM,oBAAoB,EAAE,WAMlC,CAAC;AAEF,wBAAgB,gBAAgB,IAAI,MAAM,CAMzC;AAED,wBAAsB,qBAAqB,CAC1C,GAAG,EAAE,MAAM,EACX,WAAW,GAAE,WAAkC,GAC7C,OAAO,CAAC,QAAQ,CAAC,CAgCnB;AAED,UAAU,QAAQ;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;CACvB;AAED,UAAU,SAAS;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,uBAAuB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACxC,gBAAgB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACjC,IAAI,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;IACvB,IAAI,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IAC5B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,UAAU,SAAS;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,UAAU;IAC1B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,UAAU,eAAe;IACxB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,MAAM;IACtB,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;CAC9B;AAED,MAAM,WAAW,SAAS;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;CACvB;AAwJD,wBAAgB,QAAQ,CAAC,GAAG,EAAE,UAAU,GAAG,SAAS,CAInD;AAED,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAI/C"}
+{"version":3,"file":"helpers.d.ts","sourceRoot":"","sources":["../../../../src/src/apis/hasdata/helpers.ts"],"names":[],"mappings":"AACA,OAAO,EAAe,KAAK,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAEvE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,iCAAiC,CAAC;AAG9D,eAAO,MAAM,mBAAmB,KAAK,CAAC;AAEtC,eAAO,MAAM,oBAAoB,EAAE,WAMlC,CAAC;AAEF,wBAAgB,gBAAgB,IAAI,MAAM,CAMzC;AAED,wBAAsB,qBAAqB,CAC1C,GAAG,EAAE,MAAM,EACX,WAAW,GAAE,WAAkC,GAC7C,OAAO,CAAC,QAAQ,CAAC,CAgCnB;AAED,UAAU,QAAQ;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;CACvB;AAED,UAAU,SAAS;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,uBAAuB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACxC,gBAAgB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACjC,IAAI,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;IACvB,IAAI,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IAC5B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,UAAU,SAAS;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,UAAU;IAC1B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,UAAU,eAAe;IACxB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,MAAM;IACtB,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;CAC9B;AAED,MAAM,WAAW,SAAS;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;CACvB;AAqMD,wBAAgB,QAAQ,CAAC,GAAG,EAAE,UAAU,GAAG,SAAS,CAInD;AAED,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAI/C"}

package/esm/src/apis/hasdata/helpers.js

@@ -116,8 +116,38 @@ function formatCode(block) {
     const header = `[Code${lang ? ': ' + lang : ''}]`;
     return `${header}\n${snippet.trim()}`;
 }
+function formatCitationMarkers(refIndexes) {
+    if (refIndexes.length === 0) {
+        return '';
+    }
+    return ' ' + refIndexes.map(i => `[${i + 1}]`).join('');
+}
 function parseAIResult(data, { allowNestedOverview = true } = {}) {
     const textBlocks = data.textBlocks || (allowNestedOverview ? data.aiOverview?.textBlocks : []) || [];
+    // Build reference index → source index mapping and track cited refs
+    const refs = data.references || (allowNestedOverview ? data.aiOverview?.references : []) || [];
+    const sources = [];
+    const refIndexToSourceIndex = new Map();
+    for (const r of refs) {
+        const link = r.link || r.url;
+        const title = [r.title, r.source, r.snippet].filter(Boolean).join(' - ');
+        if (link && r.index != null) {
+            // Deduplicate by URL
+            const existingIdx = sources.findIndex(s => s.url === link);
+            if (existingIdx >= 0) {
+                refIndexToSourceIndex.set(r.index, existingIdx);
+            }
+            else {
+                refIndexToSourceIndex.set(r.index, sources.length);
+                sources.push({
+                    title,
+                    url: link,
+                    domain: extractDomain(link)
+                });
+            }
+        }
+    }
+    const citedSourceIndexes = new Set();
     const parts = [];
     const handlers = {
         paragraph: (b) => cleanText(b.snippet || ''),
@@ -131,19 +161,40 @@ function parseAIResult(data, { allowNestedOverview = true } = {}) {
             continue;
         }
         const handler = handlers[btype];
+        let rendered = '';
         if (handler) {
-            const rendered = handler(block);
-            if (rendered) {
-                parts.push(rendered);
-            }
+            rendered = handler(block);
         }
         else {
             const snippet = block.snippet || '';
             if (snippet) {
-                parts.push(cleanText(snippet));
+                rendered = cleanText(snippet);
+            }
+        }
+        if (rendered) {
+            // Append citation markers and track positions
+            const refIndexes = block.referenceIndexes || [];
+            if (refIndexes.length > 0) {
+                // Map ref indexes to 1-based source indexes for display
+                const sourceIndexes = refIndexes
+                    .map(ri => refIndexToSourceIndex.get(ri))
+                    .filter((si) => si != null);
+                for (const si of sourceIndexes) {
+                    citedSourceIndexes.add(si);
+                    sources[si].positions ??= [];
+                    if (!sources[si].positions.includes(parts.length)) {
+                        sources[si].positions.push(parts.length);
+                    }
+                }
+                rendered += formatCitationMarkers(sourceIndexes.filter((v, i, a) => a.indexOf(v) === i));
             }
+            parts.push(rendered);
         }
     }
+    // Mark cited sources
+    for (const si of citedSourceIndexes) {
+        sources[si].cited = true;
+    }
     const deduped = [];
     for (const p of parts) {
         if (deduped.length === 0 || deduped[deduped.length - 1] !== p) {
@@ -155,19 +206,6 @@ function parseAIResult(data, { allowNestedOverview = true } = {}) {
         console.warn('Warning: AI answer truncated to 16000 characters');
         answer = answer.slice(0, 16000);
     }
-    const refs = data.references || (allowNestedOverview ? data.aiOverview?.references : []) || [];
-    const sources = [];
-    for (const r of refs) {
-        const link = r.link || r.url;
-        const title = [r.title, r.source, r.snippet].filter(Boolean).join(' - ');
-        if (link) {
-            sources.push({
-                title,
-                url: link,
-                domain: extractDomain(link)
-            });
-        }
-    }
     return { answer, sources };
 }
 export function parseAIO(aio) {

package/esm/src/schemas/search.schema.d.ts

@@ -1,12 +1,12 @@
 import type { z } from '../../deps/jsr.io/@zod/zod/4.3.6/src/index.js';
-import type { Source, SearchSource } from './sources.schema.js';
+import type { Source } from './sources.schema.js';
 export type ContextSize = 'low' | 'medium' | 'high';
 export type ReasoningEffort = 'low' | 'medium' | 'high';
 export interface SearchResult {
     answer: string;
+    answer_text_markdown?: string;
     sources: Array<Source>;
     searchQueries?: Array<string>;
-    searchSources?: Array<SearchSource>;
 }
 export type SearchOptions = {
     prompt: string;

package/esm/src/schemas/search.schema.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"search.schema.d.ts","sourceRoot":"","sources":["../../../src/src/schemas/search.schema.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,+CAA+C,CAAC;AAEvE,OAAO,KAAK,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAChE,MAAM,MAAM,WAAW,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;AACpD,MAAM,MAAM,eAAe,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;AAGxD,MAAM,WAAW,YAAY;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACvB,aAAa,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAC9B,aAAa,CAAC,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;CACpC;AAED,MAAM,MAAM,aAAa,GAAG;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;4FACwF;IACxF,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,UAAU,CAAC,EAAE,YAAY,GAAG,oBAAoB,CAAA;CAChD,CAAC;AAEF,MAAM,MAAM,sBAAsB,CAAC,CAAC,IAAI,aAAa,GAAG;IACvD,cAAc,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;CAC7B,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC,GAAG;IAChE,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAA;CACvB,CAAA"}
+{"version":3,"file":"search.schema.d.ts","sourceRoot":"","sources":["../../../src/src/schemas/search.schema.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,+CAA+C,CAAC;AAEvE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAClD,MAAM,MAAM,WAAW,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;AACpD,MAAM,MAAM,eAAe,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;AAExD,MAAM,WAAW,YAAY;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACvB,aAAa,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;CAC9B;AAED,MAAM,MAAM,aAAa,GAAG;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;4FACwF;IACxF,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,UAAU,CAAC,EAAE,YAAY,GAAG,oBAAoB,CAAC;CACjD,CAAC;AAEF,MAAM,MAAM,sBAAsB,CAAC,CAAC,IAAI,aAAa,GAAG;IACvD,cAAc,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;CAC7B,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC,GAAG;IAChE,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;CACxB,CAAC"}

package/esm/src/schemas/sources.schema.d.ts

@@ -3,6 +3,7 @@ export interface Source {
     url: string;
     domain: string;
     cited?: boolean;
+    snippet?: string;
     positions?: Array<number>;
 }
 /**
@@ -22,8 +23,4 @@ export interface CategorizedSource extends EnrichedSource {
     category: string | null;
     subcategory: string | null;
 }
-export interface SearchSource extends Source {
-    rank: number;
-    datePublished: string | null;
-}
 //# sourceMappingURL=sources.schema.d.ts.map

package/esm/src/schemas/sources.schema.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"sources.schema.d.ts","sourceRoot":"","sources":["../../../src/src/schemas/sources.schema.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,MAAM;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,SAAS,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;CAC1B;AAED;;;GAGG;AACH,MAAM,WAAW,cAAe,SAAQ,MAAM;IAC7C,eAAe,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAC/B,oBAAoB,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACpC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,iBAAkB,SAAQ,cAAc;IACxD,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3B;AAED,MAAM,WAAW,YAAa,SAAQ,MAAM;IAC3C,IAAI,EAAE,MAAM,CAAC;IACb,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;CAC7B"}
+{"version":3,"file":"sources.schema.d.ts","sourceRoot":"","sources":["../../../src/src/schemas/sources.schema.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,MAAM;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;CAC1B;AAED;;;GAGG;AACH,MAAM,WAAW,cAAe,SAAQ,MAAM;IAC7C,eAAe,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAC/B,oBAAoB,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACpC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;CAChC;AAED;;GAEG;AACH,MAAM,WAAW,iBAAkB,SAAQ,cAAc;IACxD,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3B"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@graphext/cuery",
-  "version": "0.9.5",
+  "version": "0.10.1",
   "description": "Cuery tools for AI-powered keyword research and brand analysis",
   "repository": {
     "type": "git",

package/script/browser.d.ts

@@ -2,7 +2,7 @@
  * Browser-safe exports for @graphext/cuery
  *
  * This module only exports types and pure functions that can safely run in the browser.
- * It excludes server-only modules like chatgptScraper, googleAds, and API functions
+ * It excludes server-only modules like llmScraper, googleAds, and API functions
  * that depend on Node.js or Deno-specific APIs.
  *
  * @module

package/script/browser.js

@@ -3,7 +3,7 @@
  * Browser-safe exports for @graphext/cuery
  *
  * This module only exports types and pure functions that can safely run in the browser.
- * It excludes server-only modules like chatgptScraper, googleAds, and API functions
+ * It excludes server-only modules like llmScraper, googleAds, and API functions
  * that depend on Node.js or Deno-specific APIs.
  *
  * @module

package/script/mod.d.ts

@@ -5,7 +5,7 @@
  */
 export * from './src/llm.js';
 export { BatchResponse } from './src/response.js';
-export { getProvider, getProviderForModel, getModelPricing, getModelInfo, calculateCost, type ModelPricing, type ModelInfo, } from './src/providers/index.js';
+export { calculateCost, getModelInfo, getModelPricing, getProvider, getProviderForModel, type ModelInfo, type ModelPricing, } from './src/providers/index.js';
 export * from './src/tools/keywords.js';
 export * from './src/tools/classifier.js';
 export * from './src/tools/funnel.js';
@@ -26,8 +26,8 @@ export * from './src/tools/scorer.js';
 export * from './src/helpers/seedKeywords.js';
 export * from './src/tools/generic.js';
 export * from './src/apis/hasdata/index.js';
-export * from './src/apis/brightdata/index.js';
-export * from './src/apis/chatgptScraper/index.js';
+export * from './src/apis/brightdata/contentScraper/index.js';
+export * from './src/apis/brightdata/llmScraper/index.js';
 export * from './src/apis/googleAds/keywordPlanner.js';
 export * from './src/schemas/index.js';
 //# sourceMappingURL=mod.d.ts.map

package/script/mod.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"mod.d.ts","sourceRoot":"","sources":["../src/mod.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,cAAc,cAAc,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAClD,OAAO,EACN,WAAW,EACX,mBAAmB,EACnB,eAAe,EACf,YAAY,EACZ,aAAa,EACb,KAAK,YAAY,EACjB,KAAK,SAAS,GACd,MAAM,0BAA0B,CAAC;AAGlC,cAAc,yBAAyB,CAAC;AACxC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,uBAAuB,CAAC;AACtC,cAAc,yBAAyB,CAAC;AACxC,cAAc,uBAAuB,CAAC;AACtC,cAAc,uBAAuB,CAAC;AACtC,cAAc,wBAAwB,CAAC;AACvC,cAAc,wBAAwB,CAAC;AACvC,cAAc,uBAAuB,CAAC;AACtC,cAAc,uBAAuB,CAAC;AACtC,cAAc,0BAA0B,CAAC;AACzC,cAAc,0BAA0B,CAAC;AACzC,cAAc,0BAA0B,CAAC;AACzC,cAAc,wBAAwB,CAAC;AACvC,cAAc,yBAAyB,CAAC;AACxC,cAAc,wBAAwB,CAAC;AACvC,cAAc,uBAAuB,CAAC;AACtC,cAAc,+BAA+B,CAAC;AAC9C,cAAc,wBAAwB,CAAC;AACvC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,gCAAgC,CAAC;AAC/C,cAAc,oCAAoC,CAAC;AACnD,cAAc,wCAAwC,CAAC;AACvD,cAAc,wBAAwB,CAAC"}
+{"version":3,"file":"mod.d.ts","sourceRoot":"","sources":["../src/mod.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,cAAc,cAAc,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAClD,OAAO,EACN,aAAa,EACb,YAAY,EACZ,eAAe,EACf,WAAW,EACX,mBAAmB,EACnB,KAAK,SAAS,EACd,KAAK,YAAY,GACjB,MAAM,0BAA0B,CAAC;AAGlC,cAAc,yBAAyB,CAAC;AACxC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,uBAAuB,CAAC;AACtC,cAAc,yBAAyB,CAAC;AACxC,cAAc,uBAAuB,CAAC;AACtC,cAAc,uBAAuB,CAAC;AACtC,cAAc,wBAAwB,CAAC;AACvC,cAAc,wBAAwB,CAAC;AACvC,cAAc,uBAAuB,CAAC;AACtC,cAAc,uBAAuB,CAAC;AACtC,cAAc,0BAA0B,CAAC;AACzC,cAAc,0BAA0B,CAAC;AACzC,cAAc,0BAA0B,CAAC;AACzC,cAAc,wBAAwB,CAAC;AACvC,cAAc,yBAAyB,CAAC;AACxC,cAAc,wBAAwB,CAAC;AACvC,cAAc,uBAAuB,CAAC;AACtC,cAAc,+BAA+B,CAAC;AAC9C,cAAc,wBAAwB,CAAC;AACvC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,+CAA+C,CAAC;AAC9D,cAAc,2CAA2C,CAAC;AAC1D,cAAc,wCAAwC,CAAC;AACvD,cAAc,wBAAwB,CAAC"}

package/script/mod.js

@@ -19,17 +19,17 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
     for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.calculateCost = exports.getModelInfo = exports.getModelPricing = exports.getProviderForModel = exports.getProvider = exports.BatchResponse = void 0;
+exports.getProviderForModel = exports.getProvider = exports.getModelPricing = exports.getModelInfo = exports.calculateCost = exports.BatchResponse = void 0;
 // Core LLM interface and types
 __exportStar(require("./src/llm.js"), exports);
 var response_js_1 = require("./src/response.js");
 Object.defineProperty(exports, "BatchResponse", { enumerable: true, get: function () { return response_js_1.BatchResponse; } });
 var index_js_1 = require("./src/providers/index.js");
+Object.defineProperty(exports, "calculateCost", { enumerable: true, get: function () { return index_js_1.calculateCost; } });
+Object.defineProperty(exports, "getModelInfo", { enumerable: true, get: function () { return index_js_1.getModelInfo; } });
+Object.defineProperty(exports, "getModelPricing", { enumerable: true, get: function () { return index_js_1.getModelPricing; } });
 Object.defineProperty(exports, "getProvider", { enumerable: true, get: function () { return index_js_1.getProvider; } });
 Object.defineProperty(exports, "getProviderForModel", { enumerable: true, get: function () { return index_js_1.getProviderForModel; } });
-Object.defineProperty(exports, "getModelPricing", { enumerable: true, get: function () { return index_js_1.getModelPricing; } });
-Object.defineProperty(exports, "getModelInfo", { enumerable: true, get: function () { return index_js_1.getModelInfo; } });
-Object.defineProperty(exports, "calculateCost", { enumerable: true, get: function () { return index_js_1.calculateCost; } });
 // Tools
 __exportStar(require("./src/tools/keywords.js"), exports);
 __exportStar(require("./src/tools/classifier.js"), exports);
@@ -51,7 +51,7 @@ __exportStar(require("./src/tools/scorer.js"), exports);
 __exportStar(require("./src/helpers/seedKeywords.js"), exports);
 __exportStar(require("./src/tools/generic.js"), exports);
 __exportStar(require("./src/apis/hasdata/index.js"), exports);
-__exportStar(require("./src/apis/brightdata/index.js"), exports);
-__exportStar(require("./src/apis/chatgptScraper/index.js"), exports);
+__exportStar(require("./src/apis/brightdata/contentScraper/index.js"), exports);
+__exportStar(require("./src/apis/brightdata/llmScraper/index.js"), exports);
 __exportStar(require("./src/apis/googleAds/keywordPlanner.js"), exports);
 __exportStar(require("./src/schemas/index.js"), exports);

package/script/src/apis/brightdata/contentScraper/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/src/apis/brightdata/contentScraper/index.ts"],"names":[],"mappings":"AAAA,cAAc,aAAa,CAAC"}

package/{esm/src/apis/brightdata → script/src/apis/brightdata/contentScraper}/scrape.d.ts

@@ -1,4 +1,4 @@
-import { type RetryConfig } from '../../helpers/async.js';
+import { type RetryConfig } from '../../../helpers/async.js';
 export interface BrightDataScrapeOptions {
     /** BrightData zone name. Defaults to "web_unlocker". */
     zone?: string;

package/script/src/apis/brightdata/contentScraper/scrape.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scrape.d.ts","sourceRoot":"","sources":["../../../../../src/src/apis/brightdata/contentScraper/scrape.ts"],"names":[],"mappings":"AACA,OAAO,EAA4B,KAAK,WAAW,EAAE,MAAM,2BAA2B,CAAC;AAavF,MAAM,WAAW,uBAAuB;IACpC,wDAAwD;IACxD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,6DAA6D;IAC7D,MAAM,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;IACxB,8DAA8D;IAC9D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,wDAAwD;IACxD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,oDAAoD;IACpD,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,kCAAkC;IAClC,WAAW,CAAC,EAAE,WAAW,CAAC;CAC7B;AAED,MAAM,WAAW,wBAAwB;IACrC,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,CAAC,EAAE,MAAM,CAAC;CACjB;AA4ED;;;GAGG;AACH,wBAAsB,gBAAgB,CAClC,GAAG,EAAE,MAAM,EACX,OAAO,GAAE,uBAA4B,GACtC,OAAO,CAAC,wBAAwB,CAAC,CAYnC;AAED;;;GAGG;AACH,wBAAsB,qBAAqB,CACvC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,EACnB,OAAO,GAAE,uBAA4B,EACrC,cAAc,GAAE,MAA+B,GAChD,OAAO,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC,CAQ1C"}

package/script/src/apis/brightdata/{scrape.js → contentScraper/scrape.js}

@@ -35,8 +35,8 @@ var __importStar = (this && this.__importStar) || (function () {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.scrapeBrightData = scrapeBrightData;
 exports.scrapeBrightDataBatch = scrapeBrightDataBatch;
-const dntShim = __importStar(require("../../../_dnt.shims.js"));
-const async_js_1 = require("../../helpers/async.js");
+const dntShim = __importStar(require("../../../../_dnt.shims.js"));
+const async_js_1 = require("../../../helpers/async.js");
 const BRIGHTDATA_CONCURRENCY = 10;
 const BRIGHTDATA_RETRY_CONFIG = {
     maxRetries: 3,

package/script/src/apis/brightdata/llmScraper/brightdata.d.ts

@@ -0,0 +1,20 @@
+import { type ProviderFunctions } from './scrape.js';
+interface BrightdataProviderConfig {
+    apiBase: string;
+    datasetId: string;
+    outputFields: Array<string>;
+    extraFields?: Array<string>;
+    targetUrl: string;
+    extraInputs?: (params: {
+        prompt: string;
+        useSearch: boolean;
+        countryISOCode: string | null;
+    }) => Record<string, unknown>;
+    providerName: string;
+    maxConcurrency: number;
+    maxPromptsPerRequest: number;
+}
+export declare function createBrightdataProvider(overrides?: Partial<BrightdataProviderConfig>): ProviderFunctions;
+export declare const brightdataProvider: ProviderFunctions;
+export {};
+//# sourceMappingURL=brightdata.d.ts.map

package/script/src/apis/brightdata/llmScraper/brightdata.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"brightdata.d.ts","sourceRoot":"","sources":["../../../../../src/src/apis/brightdata/llmScraper/brightdata.ts"],"names":[],"mappings":"AAcA,OAAO,EAA6C,KAAK,iBAAiB,EAAE,MAAM,aAAa,CAAC;AA4BhG,UAAU,wBAAwB;IACjC,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAC5B,WAAW,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,CACb,MAAM,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,OAAO,CAAC;QAAC,cAAc,EAAE,MAAM,GAAG,IAAI,CAAA;KAAE,KACzE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC7B,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,oBAAoB,EAAE,MAAM,CAAC;CAC7B;AA2DD,wBAAgB,wBAAwB,CACvC,SAAS,GAAE,OAAO,CAAC,wBAAwB,CAAM,GAC/C,iBAAiB,CAgJnB;AAMD,eAAO,MAAM,kBAAkB,EAAE,iBAA8C,CAAC"}