npm - @graphext/cuery - Versions diffs - 0.10.0 → 0.10.2 - Mend

@graphext/cuery 0.10.0 → 0.10.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (11) hide show

package/esm/src/apis/brightdata/llmScraper/index.d.ts CHANGED Viewed

@@ -2,7 +2,7 @@ import type { ModelResult } from '../../../schemas/models.schema.js';
 import { type BatchOptions } from './scrape.js';
 export type { BatchOptions };
 export type JobId = string | null;
-export type ScraperTarget = 'chatgpt' | 'aim';
+export type ScraperTarget = 'chatgpt' | 'aim' | 'generic';
 export declare function getMaxConcurrency(target?: ScraperTarget): number;
 export declare function getMaxPromptsPerRequest(target?: ScraperTarget): number;
 export declare function scrapeGPTBatch(options: BatchOptions): Promise<Array<ModelResult>>;
@@ -11,4 +11,5 @@ export declare function downloadGPTSnapshots(jobIds: Array<string | null>): Prom
 export declare function scrapeAIMBatch(options: BatchOptions): Promise<Array<ModelResult>>;
 export declare function triggerAIMBatch(options: BatchOptions): Promise<Array<string | null>>;
 export declare function downloadAIMSnapshots(jobIds: Array<string | null>): Promise<Array<ModelResult>>;
+export declare function downloadSnapshots(jobIds: Array<string | null>): Promise<Array<ModelResult>>;
 //# sourceMappingURL=index.d.ts.map

package/esm/src/apis/brightdata/llmScraper/index.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/src/apis/brightdata/llmScraper/index.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mCAAmC,CAAC;AACrE,OAAO,EAAE,KAAK,YAAY,EAAqC,MAAM,aAAa,CAAC;AAKnF,YAAY,EAAE,YAAY,EAAE,CAAC;AAC7B,MAAM,MAAM,KAAK,GAAG,MAAM,GAAG,IAAI,CAAC;AAClC,MAAM,MAAM,aAAa,GAAG,SAAS,GAAG,KAAK,CAAC;~~AAuE9C~~,wBAAgB,iBAAiB,CAAC,MAAM,GAAE,aAAyB,GAAG,MAAM,CAE3E;AAED,wBAAgB,uBAAuB,CAAC,MAAM,GAAE,aAAyB,GAAG,MAAM,CAEjF;AAGD,wBAAsB,cAAc,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAEvF;AAED,wBAAsB,eAAe,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAE1F;AAED,wBAAsB,oBAAoB,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAEpG;AAGD,wBAAsB,cAAc,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAEvF;AAED,wBAAsB,eAAe,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAE1F;AAED,wBAAsB,oBAAoB,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAEpG"}
1	+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/src/apis/brightdata/llmScraper/index.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mCAAmC,CAAC;AACrE,OAAO,EAAE,KAAK,YAAY,EAAqC,MAAM,aAAa,CAAC;AAKnF,YAAY,EAAE,YAAY,EAAE,CAAC;AAC7B,MAAM,MAAM,KAAK,GAAG,MAAM,GAAG,IAAI,CAAC;AAClC,MAAM,MAAM,aAAa,GAAG,SAAS,GAAG,KAAK,GAAG,SAAS,CAAC;AAiF1D,wBAAgB,iBAAiB,CAAC,MAAM,GAAE,aAAyB,GAAG,MAAM,CAE3E;AAED,wBAAgB,uBAAuB,CAAC,MAAM,GAAE,aAAyB,GAAG,MAAM,CAEjF;AAGD,wBAAsB,cAAc,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAEvF;AAED,wBAAsB,eAAe,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAE1F;AAED,wBAAsB,oBAAoB,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAEpG;AAGD,wBAAsB,cAAc,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAEvF;AAED,wBAAsB,eAAe,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAE1F;AAED,wBAAsB,oBAAoB,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAEpG;AAGD,wBAAsB,iBAAiB,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAEjG"}

package/esm/src/apis/brightdata/llmScraper/index.js CHANGED Viewed

@@ -57,10 +57,19 @@ function getLLMScraper(target = 'chatgpt') {
         return existingScraper;
     }
     const providerName = getProviderName();
-    const targetOptions = getTargetOptions(target);
-    const provider = providerName === 'brightdata'
-        ? createBrightdataProvider(targetOptions.brightdata)
-        : createOxylabsProvider(targetOptions.oxylabs);
+    let provider;
+    if (target === 'generic') {
+        // Generic instance: only used for download/monitor, no target-specific config needed
+        provider = providerName === 'brightdata'
+            ? createBrightdataProvider()
+            : createOxylabsProvider();
+    }
+    else {
+        const targetOptions = getTargetOptions(target);
+        provider = providerName === 'brightdata'
+            ? createBrightdataProvider(targetOptions.brightdata)
+            : createOxylabsProvider(targetOptions.oxylabs);
+    }
     const scraper = createLLMScraper(provider);
     scrapers.set(target, scraper);
     return scraper;
@@ -83,7 +92,7 @@ export async function triggerGPTBatch(options) {
     return getLLMScraper('chatgpt').triggerLLMBatch(options);
 }
 export async function downloadGPTSnapshots(jobIds) {
-    return getLLMScraper('chatgpt').downloadLLMSnapshots(jobIds);
+    return downloadSnapshots(jobIds);
 }
 // AIM scraper methods
 export async function scrapeAIMBatch(options) {
@@ -93,5 +102,9 @@ export async function triggerAIMBatch(options) {
     return getLLMScraper('aim').triggerLLMBatch(options);
 }
 export async function downloadAIMSnapshots(jobIds) {
-    return getLLMScraper('aim').downloadLLMSnapshots(jobIds);
+    return downloadSnapshots(jobIds);
+}
+// Generic download — target-agnostic, works with any job IDs
+export async function downloadSnapshots(jobIds) {
+    return getLLMScraper('generic').downloadLLMSnapshots(jobIds);
 }

package/esm/src/apis/hasdata/helpers.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"helpers.d.ts","sourceRoot":"","sources":["../../../../src/src/apis/hasdata/helpers.ts"],"names":[],"mappings":"AACA,OAAO,EAAe,KAAK,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAEvE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,iCAAiC,CAAC;AAG9D,eAAO,MAAM,mBAAmB,KAAK,CAAC;AAEtC,eAAO,MAAM,oBAAoB,EAAE,WAMlC,CAAC;AAEF,wBAAgB,gBAAgB,IAAI,MAAM,CAMzC;AAED,wBAAsB,qBAAqB,CAC1C,GAAG,EAAE,MAAM,EACX,WAAW,GAAE,WAAkC,GAC7C,OAAO,CAAC,QAAQ,CAAC,CAgCnB;AAED,UAAU,QAAQ;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;CACvB;AAED,UAAU,SAAS;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,uBAAuB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACxC,gBAAgB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACjC,IAAI,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;IACvB,IAAI,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IAC5B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,UAAU,SAAS;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,UAAU;IAC1B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,UAAU,eAAe;IACxB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,MAAM;IACtB,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;CAC9B;AAED,MAAM,WAAW,SAAS;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;CACvB;~~AAwJD~~,wBAAgB,QAAQ,CAAC,GAAG,EAAE,UAAU,GAAG,SAAS,CAInD;AAED,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAI/C"}
1	+ {"version":3,"file":"helpers.d.ts","sourceRoot":"","sources":["../../../../src/src/apis/hasdata/helpers.ts"],"names":[],"mappings":"AACA,OAAO,EAAe,KAAK,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAEvE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,iCAAiC,CAAC;AAG9D,eAAO,MAAM,mBAAmB,KAAK,CAAC;AAEtC,eAAO,MAAM,oBAAoB,EAAE,WAMlC,CAAC;AAEF,wBAAgB,gBAAgB,IAAI,MAAM,CAMzC;AAED,wBAAsB,qBAAqB,CAC1C,GAAG,EAAE,MAAM,EACX,WAAW,GAAE,WAAkC,GAC7C,OAAO,CAAC,QAAQ,CAAC,CAgCnB;AAED,UAAU,QAAQ;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;CACvB;AAED,UAAU,SAAS;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,uBAAuB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACxC,gBAAgB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACjC,IAAI,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;IACvB,IAAI,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IAC5B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,UAAU,SAAS;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,UAAU;IAC1B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,UAAU,eAAe;IACxB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,MAAM;IACtB,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;CAC9B;AAED,MAAM,WAAW,SAAS;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;CACvB;AAqMD,wBAAgB,QAAQ,CAAC,GAAG,EAAE,UAAU,GAAG,SAAS,CAInD;AAED,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAI/C"}

package/esm/src/apis/hasdata/helpers.js CHANGED Viewed

@@ -116,8 +116,38 @@ function formatCode(block) {
     const header = `[Code${lang ? ': ' + lang : ''}]`;
     return `${header}\n${snippet.trim()}`;
 }
+function formatCitationMarkers(refIndexes) {
+    if (refIndexes.length === 0) {
+        return '';
+    }
+    return ' ' + refIndexes.map(i => `[${i + 1}]`).join('');
+}
 function parseAIResult(data, { allowNestedOverview = true } = {}) {
     const textBlocks = data.textBlocks || (allowNestedOverview ? data.aiOverview?.textBlocks : []) || [];
+    // Build reference index → source index mapping and track cited refs
+    const refs = data.references || (allowNestedOverview ? data.aiOverview?.references : []) || [];
+    const sources = [];
+    const refIndexToSourceIndex = new Map();
+    for (const r of refs) {
+        const link = r.link || r.url;
+        const title = [r.title, r.source, r.snippet].filter(Boolean).join(' - ');
+        if (link && r.index != null) {
+            // Deduplicate by URL
+            const existingIdx = sources.findIndex(s => s.url === link);
+            if (existingIdx >= 0) {
+                refIndexToSourceIndex.set(r.index, existingIdx);
+            }
+            else {
+                refIndexToSourceIndex.set(r.index, sources.length);
+                sources.push({
+                    title,
+                    url: link,
+                    domain: extractDomain(link)
+                });
+            }
+        }
+    }
+    const citedSourceIndexes = new Set();
     const parts = [];
     const handlers = {
         paragraph: (b) => cleanText(b.snippet || ''),
@@ -131,19 +161,40 @@ function parseAIResult(data, { allowNestedOverview = true } = {}) {
             continue;
         }
         const handler = handlers[btype];
+        let rendered = '';
         if (handler) {
-            const rendered = handler(block);
-            if (rendered) {
-                parts.push(rendered);
-            }
+            rendered = handler(block);
         }
         else {
             const snippet = block.snippet || '';
             if (snippet) {
-                parts.push(cleanText(snippet));
+                rendered = cleanText(snippet);
+            }
+        }
+        if (rendered) {
+            // Append citation markers and track positions
+            const refIndexes = block.referenceIndexes || [];
+            if (refIndexes.length > 0) {
+                // Map ref indexes to 1-based source indexes for display
+                const sourceIndexes = refIndexes
+                    .map(ri => refIndexToSourceIndex.get(ri))
+                    .filter((si) => si != null);
+                for (const si of sourceIndexes) {
+                    citedSourceIndexes.add(si);
+                    sources[si].positions ??= [];
+                    if (!sources[si].positions.includes(parts.length)) {
+                        sources[si].positions.push(parts.length);
+                    }
+                }
+                rendered += formatCitationMarkers(sourceIndexes.filter((v, i, a) => a.indexOf(v) === i));
             }
+            parts.push(rendered);
         }
     }
+    // Mark cited sources
+    for (const si of citedSourceIndexes) {
+        sources[si].cited = true;
+    }
     const deduped = [];
     for (const p of parts) {
         if (deduped.length === 0 || deduped[deduped.length - 1] !== p) {
@@ -155,19 +206,6 @@ function parseAIResult(data, { allowNestedOverview = true } = {}) {
         console.warn('Warning: AI answer truncated to 16000 characters');
         answer = answer.slice(0, 16000);
     }
-    const refs = data.references || (allowNestedOverview ? data.aiOverview?.references : []) || [];
-    const sources = [];
-    for (const r of refs) {
-        const link = r.link || r.url;
-        const title = [r.title, r.source, r.snippet].filter(Boolean).join(' - ');
-        if (link) {
-            sources.push({
-                title,
-                url: link,
-                domain: extractDomain(link)
-            });
-        }
-    }
     return { answer, sources };
 }
 export function parseAIO(aio) {

package/package.json CHANGED Viewed

@@ -1,6 +1,6 @@
 {
   "name": "@graphext/cuery",
-  "version": "0.10.0",
+  "version": "0.10.2",
   "description": "Cuery tools for AI-powered keyword research and brand analysis",
   "repository": {
     "type": "git",

package/script/src/apis/brightdata/llmScraper/index.d.ts CHANGED Viewed

@@ -2,7 +2,7 @@ import type { ModelResult } from '../../../schemas/models.schema.js';
 import { type BatchOptions } from './scrape.js';
 export type { BatchOptions };
 export type JobId = string | null;
-export type ScraperTarget = 'chatgpt' | 'aim';
+export type ScraperTarget = 'chatgpt' | 'aim' | 'generic';
 export declare function getMaxConcurrency(target?: ScraperTarget): number;
 export declare function getMaxPromptsPerRequest(target?: ScraperTarget): number;
 export declare function scrapeGPTBatch(options: BatchOptions): Promise<Array<ModelResult>>;
@@ -11,4 +11,5 @@ export declare function downloadGPTSnapshots(jobIds: Array<string | null>): Prom
 export declare function scrapeAIMBatch(options: BatchOptions): Promise<Array<ModelResult>>;
 export declare function triggerAIMBatch(options: BatchOptions): Promise<Array<string | null>>;
 export declare function downloadAIMSnapshots(jobIds: Array<string | null>): Promise<Array<ModelResult>>;
+export declare function downloadSnapshots(jobIds: Array<string | null>): Promise<Array<ModelResult>>;
 //# sourceMappingURL=index.d.ts.map

package/script/src/apis/brightdata/llmScraper/index.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/src/apis/brightdata/llmScraper/index.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mCAAmC,CAAC;AACrE,OAAO,EAAE,KAAK,YAAY,EAAqC,MAAM,aAAa,CAAC;AAKnF,YAAY,EAAE,YAAY,EAAE,CAAC;AAC7B,MAAM,MAAM,KAAK,GAAG,MAAM,GAAG,IAAI,CAAC;AAClC,MAAM,MAAM,aAAa,GAAG,SAAS,GAAG,KAAK,CAAC;~~AAuE9C~~,wBAAgB,iBAAiB,CAAC,MAAM,GAAE,aAAyB,GAAG,MAAM,CAE3E;AAED,wBAAgB,uBAAuB,CAAC,MAAM,GAAE,aAAyB,GAAG,MAAM,CAEjF;AAGD,wBAAsB,cAAc,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAEvF;AAED,wBAAsB,eAAe,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAE1F;AAED,wBAAsB,oBAAoB,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAEpG;AAGD,wBAAsB,cAAc,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAEvF;AAED,wBAAsB,eAAe,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAE1F;AAED,wBAAsB,oBAAoB,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAEpG"}
1	+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/src/apis/brightdata/llmScraper/index.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mCAAmC,CAAC;AACrE,OAAO,EAAE,KAAK,YAAY,EAAqC,MAAM,aAAa,CAAC;AAKnF,YAAY,EAAE,YAAY,EAAE,CAAC;AAC7B,MAAM,MAAM,KAAK,GAAG,MAAM,GAAG,IAAI,CAAC;AAClC,MAAM,MAAM,aAAa,GAAG,SAAS,GAAG,KAAK,GAAG,SAAS,CAAC;AAiF1D,wBAAgB,iBAAiB,CAAC,MAAM,GAAE,aAAyB,GAAG,MAAM,CAE3E;AAED,wBAAgB,uBAAuB,CAAC,MAAM,GAAE,aAAyB,GAAG,MAAM,CAEjF;AAGD,wBAAsB,cAAc,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAEvF;AAED,wBAAsB,eAAe,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAE1F;AAED,wBAAsB,oBAAoB,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAEpG;AAGD,wBAAsB,cAAc,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAEvF;AAED,wBAAsB,eAAe,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAE1F;AAED,wBAAsB,oBAAoB,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAEpG;AAGD,wBAAsB,iBAAiB,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAEjG"}

package/script/src/apis/brightdata/llmScraper/index.js CHANGED Viewed

@@ -41,6 +41,7 @@ exports.downloadGPTSnapshots = downloadGPTSnapshots;
 exports.scrapeAIMBatch = scrapeAIMBatch;
 exports.triggerAIMBatch = triggerAIMBatch;
 exports.downloadAIMSnapshots = downloadAIMSnapshots;
+exports.downloadSnapshots = downloadSnapshots;
 /* eslint no-console: ["warn", { allow: ["log", "warn", "error"] }] */
 /**
  * LLM Scraper - Public API
@@ -100,10 +101,19 @@ function getLLMScraper(target = 'chatgpt') {
         return existingScraper;
     }
     const providerName = getProviderName();
-    const targetOptions = getTargetOptions(target);
-    const provider = providerName === 'brightdata'
-        ? (0, brightdata_js_1.createBrightdataProvider)(targetOptions.brightdata)
-        : (0, oxy_js_1.createOxylabsProvider)(targetOptions.oxylabs);
+    let provider;
+    if (target === 'generic') {
+        // Generic instance: only used for download/monitor, no target-specific config needed
+        provider = providerName === 'brightdata'
+            ? (0, brightdata_js_1.createBrightdataProvider)()
+            : (0, oxy_js_1.createOxylabsProvider)();
+    }
+    else {
+        const targetOptions = getTargetOptions(target);
+        provider = providerName === 'brightdata'
+            ? (0, brightdata_js_1.createBrightdataProvider)(targetOptions.brightdata)
+            : (0, oxy_js_1.createOxylabsProvider)(targetOptions.oxylabs);
+    }
     const scraper = (0, scrape_js_1.createLLMScraper)(provider);
     scrapers.set(target, scraper);
     return scraper;
@@ -126,7 +136,7 @@ async function triggerGPTBatch(options) {
     return getLLMScraper('chatgpt').triggerLLMBatch(options);
 }
 async function downloadGPTSnapshots(jobIds) {
-    return getLLMScraper('chatgpt').downloadLLMSnapshots(jobIds);
+    return downloadSnapshots(jobIds);
 }
 // AIM scraper methods
 async function scrapeAIMBatch(options) {
@@ -136,5 +146,9 @@ async function triggerAIMBatch(options) {
     return getLLMScraper('aim').triggerLLMBatch(options);
 }
 async function downloadAIMSnapshots(jobIds) {
-    return getLLMScraper('aim').downloadLLMSnapshots(jobIds);
+    return downloadSnapshots(jobIds);
+}
+// Generic download — target-agnostic, works with any job IDs
+async function downloadSnapshots(jobIds) {
+    return getLLMScraper('generic').downloadLLMSnapshots(jobIds);
 }

package/script/src/apis/hasdata/helpers.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"helpers.d.ts","sourceRoot":"","sources":["../../../../src/src/apis/hasdata/helpers.ts"],"names":[],"mappings":"AACA,OAAO,EAAe,KAAK,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAEvE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,iCAAiC,CAAC;AAG9D,eAAO,MAAM,mBAAmB,KAAK,CAAC;AAEtC,eAAO,MAAM,oBAAoB,EAAE,WAMlC,CAAC;AAEF,wBAAgB,gBAAgB,IAAI,MAAM,CAMzC;AAED,wBAAsB,qBAAqB,CAC1C,GAAG,EAAE,MAAM,EACX,WAAW,GAAE,WAAkC,GAC7C,OAAO,CAAC,QAAQ,CAAC,CAgCnB;AAED,UAAU,QAAQ;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;CACvB;AAED,UAAU,SAAS;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,uBAAuB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACxC,gBAAgB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACjC,IAAI,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;IACvB,IAAI,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IAC5B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,UAAU,SAAS;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,UAAU;IAC1B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,UAAU,eAAe;IACxB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,MAAM;IACtB,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;CAC9B;AAED,MAAM,WAAW,SAAS;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;CACvB;~~AAwJD~~,wBAAgB,QAAQ,CAAC,GAAG,EAAE,UAAU,GAAG,SAAS,CAInD;AAED,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAI/C"}
1	+ {"version":3,"file":"helpers.d.ts","sourceRoot":"","sources":["../../../../src/src/apis/hasdata/helpers.ts"],"names":[],"mappings":"AACA,OAAO,EAAe,KAAK,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAEvE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,iCAAiC,CAAC;AAG9D,eAAO,MAAM,mBAAmB,KAAK,CAAC;AAEtC,eAAO,MAAM,oBAAoB,EAAE,WAMlC,CAAC;AAEF,wBAAgB,gBAAgB,IAAI,MAAM,CAMzC;AAED,wBAAsB,qBAAqB,CAC1C,GAAG,EAAE,MAAM,EACX,WAAW,GAAE,WAAkC,GAC7C,OAAO,CAAC,QAAQ,CAAC,CAgCnB;AAED,UAAU,QAAQ;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;CACvB;AAED,UAAU,SAAS;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,uBAAuB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACxC,gBAAgB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACjC,IAAI,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;IACvB,IAAI,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IAC5B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,UAAU,SAAS;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,UAAU;IAC1B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,UAAU,eAAe;IACxB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,MAAM;IACtB,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;CAC9B;AAED,MAAM,WAAW,SAAS;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;CACvB;AAqMD,wBAAgB,QAAQ,CAAC,GAAG,EAAE,UAAU,GAAG,SAAS,CAInD;AAED,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAI/C"}

package/script/src/apis/hasdata/helpers.js CHANGED Viewed

@@ -156,8 +156,38 @@ function formatCode(block) {
     const header = `[Code${lang ? ': ' + lang : ''}]`;
     return `${header}\n${snippet.trim()}`;
 }
+function formatCitationMarkers(refIndexes) {
+    if (refIndexes.length === 0) {
+        return '';
+    }
+    return ' ' + refIndexes.map(i => `[${i + 1}]`).join('');
+}
 function parseAIResult(data, { allowNestedOverview = true } = {}) {
     const textBlocks = data.textBlocks || (allowNestedOverview ? data.aiOverview?.textBlocks : []) || [];
+    // Build reference index → source index mapping and track cited refs
+    const refs = data.references || (allowNestedOverview ? data.aiOverview?.references : []) || [];
+    const sources = [];
+    const refIndexToSourceIndex = new Map();
+    for (const r of refs) {
+        const link = r.link || r.url;
+        const title = [r.title, r.source, r.snippet].filter(Boolean).join(' - ');
+        if (link && r.index != null) {
+            // Deduplicate by URL
+            const existingIdx = sources.findIndex(s => s.url === link);
+            if (existingIdx >= 0) {
+                refIndexToSourceIndex.set(r.index, existingIdx);
+            }
+            else {
+                refIndexToSourceIndex.set(r.index, sources.length);
+                sources.push({
+                    title,
+                    url: link,
+                    domain: (0, urls_js_1.extractDomain)(link)
+                });
+            }
+        }
+    }
+    const citedSourceIndexes = new Set();
     const parts = [];
     const handlers = {
         paragraph: (b) => cleanText(b.snippet || ''),
@@ -171,19 +201,40 @@ function parseAIResult(data, { allowNestedOverview = true } = {}) {
             continue;
         }
         const handler = handlers[btype];
+        let rendered = '';
         if (handler) {
-            const rendered = handler(block);
-            if (rendered) {
-                parts.push(rendered);
-            }
+            rendered = handler(block);
         }
         else {
             const snippet = block.snippet || '';
             if (snippet) {
-                parts.push(cleanText(snippet));
+                rendered = cleanText(snippet);
+            }
+        }
+        if (rendered) {
+            // Append citation markers and track positions
+            const refIndexes = block.referenceIndexes || [];
+            if (refIndexes.length > 0) {
+                // Map ref indexes to 1-based source indexes for display
+                const sourceIndexes = refIndexes
+                    .map(ri => refIndexToSourceIndex.get(ri))
+                    .filter((si) => si != null);
+                for (const si of sourceIndexes) {
+                    citedSourceIndexes.add(si);
+                    sources[si].positions ??= [];
+                    if (!sources[si].positions.includes(parts.length)) {
+                        sources[si].positions.push(parts.length);
+                    }
+                }
+                rendered += formatCitationMarkers(sourceIndexes.filter((v, i, a) => a.indexOf(v) === i));
             }
+            parts.push(rendered);
         }
     }
+    // Mark cited sources
+    for (const si of citedSourceIndexes) {
+        sources[si].cited = true;
+    }
     const deduped = [];
     for (const p of parts) {
         if (deduped.length === 0 || deduped[deduped.length - 1] !== p) {
@@ -195,19 +246,6 @@ function parseAIResult(data, { allowNestedOverview = true } = {}) {
         console.warn('Warning: AI answer truncated to 16000 characters');
         answer = answer.slice(0, 16000);
     }
-    const refs = data.references || (allowNestedOverview ? data.aiOverview?.references : []) || [];
-    const sources = [];
-    for (const r of refs) {
-        const link = r.link || r.url;
-        const title = [r.title, r.source, r.snippet].filter(Boolean).join(' - ');
-        if (link) {
-            sources.push({
-                title,
-                url: link,
-                domain: (0, urls_js_1.extractDomain)(link)
-            });
-        }
-    }
     return { answer, sources };
 }
 function parseAIO(aio) {