@graphext/cuery 0.10.1 → 0.10.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (27) hide show
  1. package/esm/src/apis/brightdata/llmScraper/brightdata.js +3 -3
  2. package/esm/src/apis/brightdata/llmScraper/index.d.ts +2 -1
  3. package/esm/src/apis/brightdata/llmScraper/index.d.ts.map +1 -1
  4. package/esm/src/apis/brightdata/llmScraper/index.js +19 -6
  5. package/esm/src/apis/brightdata/llmScraper/oxy.d.ts.map +1 -1
  6. package/esm/src/apis/brightdata/llmScraper/oxy.js +3 -3
  7. package/esm/src/apis/brightdata/llmScraper/scrape.d.ts +1 -2
  8. package/esm/src/apis/brightdata/llmScraper/scrape.d.ts.map +1 -1
  9. package/esm/src/apis/brightdata/llmScraper/scrape.js +8 -3
  10. package/esm/src/apis/hasdata/helpers.d.ts.map +1 -1
  11. package/esm/src/apis/hasdata/helpers.js +15 -5
  12. package/esm/src/schemas/search.schema.d.ts +1 -1
  13. package/esm/src/schemas/search.schema.d.ts.map +1 -1
  14. package/package.json +1 -1
  15. package/script/src/apis/brightdata/llmScraper/brightdata.js +2 -2
  16. package/script/src/apis/brightdata/llmScraper/index.d.ts +2 -1
  17. package/script/src/apis/brightdata/llmScraper/index.d.ts.map +1 -1
  18. package/script/src/apis/brightdata/llmScraper/index.js +20 -6
  19. package/script/src/apis/brightdata/llmScraper/oxy.d.ts.map +1 -1
  20. package/script/src/apis/brightdata/llmScraper/oxy.js +2 -2
  21. package/script/src/apis/brightdata/llmScraper/scrape.d.ts +1 -2
  22. package/script/src/apis/brightdata/llmScraper/scrape.d.ts.map +1 -1
  23. package/script/src/apis/brightdata/llmScraper/scrape.js +9 -4
  24. package/script/src/apis/hasdata/helpers.d.ts.map +1 -1
  25. package/script/src/apis/hasdata/helpers.js +15 -5
  26. package/script/src/schemas/search.schema.d.ts +1 -1
  27. package/script/src/schemas/search.schema.d.ts.map +1 -1
package/esm/src/apis/brightdata/llmScraper/brightdata.js CHANGED
@@ -9,7 +9,7 @@
9
9
  */
10
10
  import * as dntShim from "../../../../_dnt.shims.js";
11
11
  import { sleep, withRetries } from '../../../helpers/async.js';
12
- import { buildSources, cleanAnswer, getAbortSignal } from './scrape.js';
12
+ import { parseSources, cleanAnswer, getAbortSignal } from './scrape.js';
13
13
  const DEFAULT_BRIGHTDATA_PROVIDER_CONFIG = {
14
14
  apiBase: 'https://api.brightdata.com',
15
15
  datasetId: 'gd_m7aof0k82r803d5bjm',
@@ -161,8 +161,8 @@ export function createBrightdataProvider(overrides = {}) {
161
161
  return {
162
162
  prompt: response.prompt,
163
163
  answer: answerText,
164
- answer_text_markdown: answerTextMarkdown,
165
- sources: buildSources(response.citations ?? [], response.links_attached ?? []),
164
+ answerMarkdown: answerTextMarkdown,
165
+ sources: parseSources(response.citations ?? [], response.links_attached ?? []),
166
166
  searchQueries: response.web_search_query || [],
167
167
  };
168
168
  }
package/esm/src/apis/brightdata/llmScraper/index.d.ts CHANGED
@@ -2,7 +2,7 @@ import type { ModelResult } from '../../../schemas/models.schema.js';
2
2
  import { type BatchOptions } from './scrape.js';
3
3
  export type { BatchOptions };
4
4
  export type JobId = string | null;
5
- export type ScraperTarget = 'chatgpt' | 'aim';
5
+ export type ScraperTarget = 'chatgpt' | 'aim' | 'generic';
6
6
  export declare function getMaxConcurrency(target?: ScraperTarget): number;
7
7
  export declare function getMaxPromptsPerRequest(target?: ScraperTarget): number;
8
8
  export declare function scrapeGPTBatch(options: BatchOptions): Promise<Array<ModelResult>>;
@@ -11,4 +11,5 @@ export declare function downloadGPTSnapshots(jobIds: Array<string | null>): Prom
11
11
  export declare function scrapeAIMBatch(options: BatchOptions): Promise<Array<ModelResult>>;
12
12
  export declare function triggerAIMBatch(options: BatchOptions): Promise<Array<string | null>>;
13
13
  export declare function downloadAIMSnapshots(jobIds: Array<string | null>): Promise<Array<ModelResult>>;
14
+ export declare function downloadSnapshots(jobIds: Array<string | null>): Promise<Array<ModelResult>>;
14
15
  //# sourceMappingURL=index.d.ts.map
package/esm/src/apis/brightdata/llmScraper/index.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/src/apis/brightdata/llmScraper/index.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mCAAmC,CAAC;AACrE,OAAO,EAAE,KAAK,YAAY,EAAqC,MAAM,aAAa,CAAC;AAKnF,YAAY,EAAE,YAAY,EAAE,CAAC;AAC7B,MAAM,MAAM,KAAK,GAAG,MAAM,GAAG,IAAI,CAAC;AAClC,MAAM,MAAM,aAAa,GAAG,SAAS,GAAG,KAAK,CAAC;AAuE9C,wBAAgB,iBAAiB,CAAC,MAAM,GAAE,aAAyB,GAAG,MAAM,CAE3E;AAED,wBAAgB,uBAAuB,CAAC,MAAM,GAAE,aAAyB,GAAG,MAAM,CAEjF;AAGD,wBAAsB,cAAc,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAEvF;AAED,wBAAsB,eAAe,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAE1F;AAED,wBAAsB,oBAAoB,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAEpG;AAGD,wBAAsB,cAAc,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAEvF;AAED,wBAAsB,eAAe,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAE1F;AAED,wBAAsB,oBAAoB,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAEpG"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/src/apis/brightdata/llmScraper/index.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mCAAmC,CAAC;AACrE,OAAO,EAAE,KAAK,YAAY,EAAqC,MAAM,aAAa,CAAC;AAKnF,YAAY,EAAE,YAAY,EAAE,CAAC;AAC7B,MAAM,MAAM,KAAK,GAAG,MAAM,GAAG,IAAI,CAAC;AAClC,MAAM,MAAM,aAAa,GAAG,SAAS,GAAG,KAAK,GAAG,SAAS,CAAC;AAiF1D,wBAAgB,iBAAiB,CAAC,MAAM,GAAE,aAAyB,GAAG,MAAM,CAE3E;AAED,wBAAgB,uBAAuB,CAAC,MAAM,GAAE,aAAyB,GAAG,MAAM,CAEjF;AAGD,wBAAsB,cAAc,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAEvF;AAED,wBAAsB,eAAe,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAE1F;AAED,wBAAsB,oBAAoB,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAEpG;AAGD,wBAAsB,cAAc,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAEvF;AAED,wBAAsB,eAAe,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAE1F;AAED,wBAAsB,oBAAoB,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAEpG;AAGD,wBAAsB,iBAAiB,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAEjG"}
package/esm/src/apis/brightdata/llmScraper/index.js CHANGED
@@ -57,10 +57,19 @@ function getLLMScraper(target = 'chatgpt') {
57
57
  return existingScraper;
58
58
  }
59
59
  const providerName = getProviderName();
60
- const targetOptions = getTargetOptions(target);
61
- const provider = providerName === 'brightdata'
62
- ? createBrightdataProvider(targetOptions.brightdata)
63
- : createOxylabsProvider(targetOptions.oxylabs);
60
+ let provider;
61
+ if (target === 'generic') {
62
+ // Generic instance: only used for download/monitor, no target-specific config needed
63
+ provider = providerName === 'brightdata'
64
+ ? createBrightdataProvider()
65
+ : createOxylabsProvider();
66
+ }
67
+ else {
68
+ const targetOptions = getTargetOptions(target);
69
+ provider = providerName === 'brightdata'
70
+ ? createBrightdataProvider(targetOptions.brightdata)
71
+ : createOxylabsProvider(targetOptions.oxylabs);
72
+ }
64
73
  const scraper = createLLMScraper(provider);
65
74
  scrapers.set(target, scraper);
66
75
  return scraper;
@@ -83,7 +92,7 @@ export async function triggerGPTBatch(options) {
83
92
  return getLLMScraper('chatgpt').triggerLLMBatch(options);
84
93
  }
85
94
  export async function downloadGPTSnapshots(jobIds) {
86
- return getLLMScraper('chatgpt').downloadLLMSnapshots(jobIds);
95
+ return downloadSnapshots(jobIds);
87
96
  }
88
97
  // AIM scraper methods
89
98
  export async function scrapeAIMBatch(options) {
@@ -93,5 +102,9 @@ export async function triggerAIMBatch(options) {
93
102
  return getLLMScraper('aim').triggerLLMBatch(options);
94
103
  }
95
104
  export async function downloadAIMSnapshots(jobIds) {
96
- return getLLMScraper('aim').downloadLLMSnapshots(jobIds);
105
+ return downloadSnapshots(jobIds);
106
+ }
107
+ // Generic download — target-agnostic, works with any job IDs
108
+ export async function downloadSnapshots(jobIds) {
109
+ return getLLMScraper('generic').downloadLLMSnapshots(jobIds);
97
110
  }
package/esm/src/apis/brightdata/llmScraper/oxy.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"oxy.d.ts","sourceRoot":"","sources":["../../../../../src/src/apis/brightdata/llmScraper/oxy.ts"],"names":[],"mappings":"AAeA,OAAO,EAA6C,KAAK,iBAAiB,EAAE,MAAM,aAAa,CAAC;AA2BhG,UAAU,qBAAqB;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC;IAC7B,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,oBAAoB,EAAE,MAAM,CAAC;CAC7B;AAyCD,wBAAgB,qBAAqB,CAAC,SAAS,GAAE,OAAO,CAAC,qBAAqB,CAAM,GAAG,iBAAiB,CAyJvG;AAMD,eAAO,MAAM,eAAe,EAAE,iBAA2C,CAAC"}
1
+ {"version":3,"file":"oxy.d.ts","sourceRoot":"","sources":["../../../../../src/src/apis/brightdata/llmScraper/oxy.ts"],"names":[],"mappings":"AAeA,OAAO,EAA6C,KAAK,iBAAiB,EAAE,MAAM,aAAa,CAAC;AA0BhG,UAAU,qBAAqB;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC;IAC7B,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,oBAAoB,EAAE,MAAM,CAAC;CAC7B;AAyCD,wBAAgB,qBAAqB,CAAC,SAAS,GAAE,OAAO,CAAC,qBAAqB,CAAM,GAAG,iBAAiB,CAyJvG;AAMD,eAAO,MAAM,eAAe,EAAE,iBAA2C,CAAC"}
package/esm/src/apis/brightdata/llmScraper/oxy.js CHANGED
@@ -9,7 +9,7 @@
9
9
  */
10
10
  import * as dntShim from "../../../../_dnt.shims.js";
11
11
  import { sleep, withRetries } from '../../../helpers/async.js';
12
- import { buildSources, cleanAnswer, getAbortSignal } from './scrape.js';
12
+ import { parseSources, cleanAnswer, getAbortSignal } from './scrape.js';
13
13
  const DEFAULT_OXYLABS_PROVIDER_CONFIG = {
14
14
  apiBase: 'https://data.oxylabs.io/v1',
15
15
  source: 'chatgpt',
@@ -150,8 +150,8 @@ export function createOxylabsProvider(overrides = {}) {
150
150
  return {
151
151
  prompt: content.prompt || '',
152
152
  answer: answerText,
153
- answer_text_markdown: answerTextMarkdown,
154
- sources: buildSources(citations),
153
+ answerMarkdown: answerTextMarkdown,
154
+ sources: parseSources(citations),
155
155
  searchQueries: [],
156
156
  };
157
157
  }
package/esm/src/apis/brightdata/llmScraper/scrape.d.ts CHANGED
@@ -23,11 +23,10 @@ export interface LLMScraper {
23
23
  }
24
24
  export declare function getAbortSignal(): AbortSignal | undefined;
25
25
  export declare function cleanAnswer(answer: string): string;
26
- export declare function buildSources(citations: Array<{
26
+ export declare function parseSources(citations: Array<{
27
27
  url: string;
28
28
  title?: string;
29
29
  description?: string;
30
- text?: string;
31
30
  cited?: boolean;
32
31
  }>, linksAttached?: Array<{
33
32
  url?: string;
package/esm/src/apis/brightdata/llmScraper/scrape.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"scrape.d.ts","sourceRoot":"","sources":["../../../../../src/src/apis/brightdata/llmScraper/scrape.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mCAAmC,CAAC;AACrE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oCAAoC,CAAC;AAOjE,MAAM,WAAW,YAAY;IAC5B,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACvB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/B;AAED,MAAM,WAAW,iBAAiB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,EAAE,MAAM,CAAC;IACvB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,UAAU,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,GAAG,IAAI,KAAK,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAC1G,UAAU,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IAChD,WAAW,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IACjD,iBAAiB,EAAE,CAAC,GAAG,EAAE,OAAO,KAAK,WAAW,GAAG,IAAI,CAAC;CACxD;AAED,MAAM,WAAW,UAAU;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,cAAc,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC;IACvE,eAAe,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC;IAC1E,oBAAoB,EAAE,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC;CACpF;AAMD,wBAAgB,cAAc,IAAI,WAAW,GAAG,SAAS,CAExD;AAED,wBAAgB,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAMlD;AA+BD,wBAAgB,YAAY,CAC3B,SAAS,EAAE,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,OAAO,CAAA;CAAE,CAAC,EACvG,aAAa,GAAE,KAAK,CAAC;IAAE,GAAG,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,CAAM,GAC3E,KAAK,CAAC,MAAM,CAAC,CAoFf;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,YAAY,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,GAAG,WAAW,CAU5G;AAMD,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,iBAAiB,GAAG,UAAU,CAkExE"}
1
+ {"version":3,"file":"scrape.d.ts","sourceRoot":"","sources":["../../../../../src/src/apis/brightdata/llmScraper/scrape.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mCAAmC,CAAC;AACrE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oCAAoC,CAAC;AAOjE,MAAM,WAAW,YAAY;IAC5B,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACvB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/B;AAED,MAAM,WAAW,iBAAiB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,EAAE,MAAM,CAAC;IACvB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,UAAU,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,GAAG,IAAI,KAAK,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAC1G,UAAU,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IAChD,WAAW,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IACjD,iBAAiB,EAAE,CAAC,GAAG,EAAE,OAAO,KAAK,WAAW,GAAG,IAAI,CAAC;CACxD;AAED,MAAM,WAAW,UAAU;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,cAAc,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC;IACvE,eAAe,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC;IAC1E,oBAAoB,EAAE,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC;CACpF;AAMD,wBAAgB,cAAc,IAAI,WAAW,GAAG,SAAS,CAExD;AAED,wBAAgB,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAMlD;AA+BD,wBAAgB,YAAY,CAC3B,SAAS,EAAE,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,OAAO,CAAA;CAAE,CAAC,EACxF,aAAa,GAAE,KAAK,CAAC;IAAE,GAAG,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,CAAM,GAC3E,KAAK,CAAC,MAAM,CAAC,CAyFf;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,YAAY,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,GAAG,WAAW,CAU5G;AAMD,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,iBAAiB,GAAG,UAAU,CAkExE"}
package/esm/src/apis/brightdata/llmScraper/scrape.js CHANGED
@@ -49,7 +49,7 @@ function hasExtraUrlInfo(current, candidate) {
49
49
  return false;
50
50
  }
51
51
  }
52
- export function buildSources(citations, linksAttached = []) {
52
+ export function parseSources(citations, linksAttached = []) {
53
53
  const sources = [];
54
54
  const sourcesByKey = new Map();
55
55
  const upsertSource = (url, initialTitle, cited) => {
@@ -97,11 +97,15 @@ export function buildSources(citations, linksAttached = []) {
97
97
  continue;
98
98
  const key = urlMergeKey(citation.url);
99
99
  const existing = sourcesByKey.get(key);
100
- const title = citation.title || citation.description || citation.text || '';
100
+ const title = citation.title ?? '';
101
+ const snippet = citation.description;
101
102
  if (existing) {
102
103
  if (title) {
103
104
  existing.title = title;
104
105
  }
106
+ if (snippet) {
107
+ existing.snippet = snippet;
108
+ }
105
109
  existing.cited = existing.cited || citation.cited;
106
110
  // Append extra fragment/params from citation
107
111
  if (hasExtraUrlInfo(existing.url, citation.url)) {
@@ -111,6 +115,7 @@ export function buildSources(citations, linksAttached = []) {
111
115
  }
112
116
  const source = {
113
117
  title,
118
+ snippet,
114
119
  url: citation.url,
115
120
  domain: extractDomain(citation.url),
116
121
  cited: citation.cited,
@@ -134,7 +139,7 @@ export function emptyModelResult(providerName, errorMessage, context) {
134
139
  return {
135
140
  prompt: '',
136
141
  answer: '',
137
- answer_text_markdown: '',
142
+ answerMarkdown: '',
138
143
  sources: [],
139
144
  };
140
145
  }
package/esm/src/apis/hasdata/helpers.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"helpers.d.ts","sourceRoot":"","sources":["../../../../src/src/apis/hasdata/helpers.ts"],"names":[],"mappings":"AACA,OAAO,EAAe,KAAK,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAEvE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,iCAAiC,CAAC;AAG9D,eAAO,MAAM,mBAAmB,KAAK,CAAC;AAEtC,eAAO,MAAM,oBAAoB,EAAE,WAMlC,CAAC;AAEF,wBAAgB,gBAAgB,IAAI,MAAM,CAMzC;AAED,wBAAsB,qBAAqB,CAC1C,GAAG,EAAE,MAAM,EACX,WAAW,GAAE,WAAkC,GAC7C,OAAO,CAAC,QAAQ,CAAC,CAgCnB;AAED,UAAU,QAAQ;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;CACvB;AAED,UAAU,SAAS;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,uBAAuB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACxC,gBAAgB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACjC,IAAI,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;IACvB,IAAI,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IAC5B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,UAAU,SAAS;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,UAAU;IAC1B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,UAAU,eAAe;IACxB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,MAAM;IACtB,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;CAC9B;AAED,MAAM,WAAW,SAAS;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;CACvB;AAqMD,wBAAgB,QAAQ,CAAC,GAAG,EAAE,UAAU,GAAG,SAAS,CAInD;AAED,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAI/C"}
1
+ {"version":3,"file":"helpers.d.ts","sourceRoot":"","sources":["../../../../src/src/apis/hasdata/helpers.ts"],"names":[],"mappings":"AACA,OAAO,EAAe,KAAK,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAEvE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,iCAAiC,CAAC;AAG9D,eAAO,MAAM,mBAAmB,KAAK,CAAC;AAEtC,eAAO,MAAM,oBAAoB,EAAE,WAMlC,CAAC;AAEF,wBAAgB,gBAAgB,IAAI,MAAM,CAMzC;AAED,wBAAsB,qBAAqB,CAC1C,GAAG,EAAE,MAAM,EACX,WAAW,GAAE,WAAkC,GAC7C,OAAO,CAAC,QAAQ,CAAC,CAgCnB;AAED,UAAU,QAAQ;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;CACvB;AAED,UAAU,SAAS;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,uBAAuB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACxC,gBAAgB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACjC,IAAI,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;IACvB,IAAI,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IAC5B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,UAAU,SAAS;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,UAAU;IAC1B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,UAAU,eAAe;IACxB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,MAAM;IACtB,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;CAC9B;AAED,MAAM,WAAW,SAAS;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;CACvB;AA6MD,wBAAgB,QAAQ,CAAC,GAAG,EAAE,UAAU,GAAG,SAAS,CAInD;AAED,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAI/C"}
package/esm/src/apis/hasdata/helpers.js CHANGED
@@ -130,17 +130,25 @@ function parseAIResult(data, { allowNestedOverview = true } = {}) {
130
130
  const refIndexToSourceIndex = new Map();
131
131
  for (const r of refs) {
132
132
  const link = r.link || r.url;
133
- const title = [r.title, r.source, r.snippet].filter(Boolean).join(' - ');
133
+ const title = [r.title, r.source].filter(Boolean).join(' - ');
134
+ const snippet = cleanText(r.snippet || '') || undefined;
134
135
  if (link && r.index != null) {
135
136
  // Deduplicate by URL
136
137
  const existingIdx = sources.findIndex(s => s.url === link);
137
138
  if (existingIdx >= 0) {
138
139
  refIndexToSourceIndex.set(r.index, existingIdx);
140
+ if (!sources[existingIdx].title && title) {
141
+ sources[existingIdx].title = title;
142
+ }
143
+ if (!sources[existingIdx].snippet && snippet) {
144
+ sources[existingIdx].snippet = snippet;
145
+ }
139
146
  }
140
147
  else {
141
148
  refIndexToSourceIndex.set(r.index, sources.length);
142
149
  sources.push({
143
150
  title,
151
+ snippet,
144
152
  url: link,
145
153
  domain: extractDomain(link)
146
154
  });
@@ -179,14 +187,16 @@ function parseAIResult(data, { allowNestedOverview = true } = {}) {
179
187
  const sourceIndexes = refIndexes
180
188
  .map(ri => refIndexToSourceIndex.get(ri))
181
189
  .filter((si) => si != null);
182
- for (const si of sourceIndexes) {
190
+ const uniqueSourceIndexes = sourceIndexes.filter((v, i, a) => a.indexOf(v) === i);
191
+ for (const si of uniqueSourceIndexes) {
183
192
  citedSourceIndexes.add(si);
184
193
  sources[si].positions ??= [];
185
- if (!sources[si].positions.includes(parts.length)) {
186
- sources[si].positions.push(parts.length);
194
+ const citationNumber = si + 1;
195
+ if (!sources[si].positions.includes(citationNumber)) {
196
+ sources[si].positions.push(citationNumber);
187
197
  }
188
198
  }
189
- rendered += formatCitationMarkers(sourceIndexes.filter((v, i, a) => a.indexOf(v) === i));
199
+ rendered += formatCitationMarkers(uniqueSourceIndexes);
190
200
  }
191
201
  parts.push(rendered);
192
202
  }
package/esm/src/schemas/search.schema.d.ts CHANGED
@@ -4,9 +4,9 @@ export type ContextSize = 'low' | 'medium' | 'high';
4
4
  export type ReasoningEffort = 'low' | 'medium' | 'high';
5
5
  export interface SearchResult {
6
6
  answer: string;
7
- answer_text_markdown?: string;
8
7
  sources: Array<Source>;
9
8
  searchQueries?: Array<string>;
9
+ answerMarkdown?: string;
10
10
  }
11
11
  export type SearchOptions = {
12
12
  prompt: string;
package/esm/src/schemas/search.schema.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"search.schema.d.ts","sourceRoot":"","sources":["../../../src/src/schemas/search.schema.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,+CAA+C,CAAC;AAEvE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAClD,MAAM,MAAM,WAAW,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;AACpD,MAAM,MAAM,eAAe,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;AAExD,MAAM,WAAW,YAAY;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACvB,aAAa,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;CAC9B;AAED,MAAM,MAAM,aAAa,GAAG;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;4FACwF;IACxF,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,UAAU,CAAC,EAAE,YAAY,GAAG,oBAAoB,CAAC;CACjD,CAAC;AAEF,MAAM,MAAM,sBAAsB,CAAC,CAAC,IAAI,aAAa,GAAG;IACvD,cAAc,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;CAC7B,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC,GAAG;IAChE,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;CACxB,CAAC"}
1
+ {"version":3,"file":"search.schema.d.ts","sourceRoot":"","sources":["../../../src/src/schemas/search.schema.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,+CAA+C,CAAC;AAEvE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAClD,MAAM,MAAM,WAAW,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;AACpD,MAAM,MAAM,eAAe,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;AAExD,MAAM,WAAW,YAAY;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACvB,aAAa,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAC9B,cAAc,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,MAAM,aAAa,GAAG;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;4FACwF;IACxF,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,UAAU,CAAC,EAAE,YAAY,GAAG,oBAAoB,CAAC;CACjD,CAAC;AAEF,MAAM,MAAM,sBAAsB,CAAC,CAAC,IAAI,aAAa,GAAG;IACvD,cAAc,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;CAC7B,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC,GAAG;IAChE,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;CACxB,CAAC"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@graphext/cuery",
3
- "version": "0.10.1",
3
+ "version": "0.10.3",
4
4
  "description": "Cuery tools for AI-powered keyword research and brand analysis",
5
5
  "repository": {
6
6
  "type": "git",
package/script/src/apis/brightdata/llmScraper/brightdata.js CHANGED
@@ -198,8 +198,8 @@ function createBrightdataProvider(overrides = {}) {
198
198
  return {
199
199
  prompt: response.prompt,
200
200
  answer: answerText,
201
- answer_text_markdown: answerTextMarkdown,
202
- sources: (0, scrape_js_1.buildSources)(response.citations ?? [], response.links_attached ?? []),
201
+ answerMarkdown: answerTextMarkdown,
202
+ sources: (0, scrape_js_1.parseSources)(response.citations ?? [], response.links_attached ?? []),
203
203
  searchQueries: response.web_search_query || [],
204
204
  };
205
205
  }
package/script/src/apis/brightdata/llmScraper/index.d.ts CHANGED
@@ -2,7 +2,7 @@ import type { ModelResult } from '../../../schemas/models.schema.js';
2
2
  import { type BatchOptions } from './scrape.js';
3
3
  export type { BatchOptions };
4
4
  export type JobId = string | null;
5
- export type ScraperTarget = 'chatgpt' | 'aim';
5
+ export type ScraperTarget = 'chatgpt' | 'aim' | 'generic';
6
6
  export declare function getMaxConcurrency(target?: ScraperTarget): number;
7
7
  export declare function getMaxPromptsPerRequest(target?: ScraperTarget): number;
8
8
  export declare function scrapeGPTBatch(options: BatchOptions): Promise<Array<ModelResult>>;
@@ -11,4 +11,5 @@ export declare function downloadGPTSnapshots(jobIds: Array<string | null>): Prom
11
11
  export declare function scrapeAIMBatch(options: BatchOptions): Promise<Array<ModelResult>>;
12
12
  export declare function triggerAIMBatch(options: BatchOptions): Promise<Array<string | null>>;
13
13
  export declare function downloadAIMSnapshots(jobIds: Array<string | null>): Promise<Array<ModelResult>>;
14
+ export declare function downloadSnapshots(jobIds: Array<string | null>): Promise<Array<ModelResult>>;
14
15
  //# sourceMappingURL=index.d.ts.map
package/script/src/apis/brightdata/llmScraper/index.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/src/apis/brightdata/llmScraper/index.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mCAAmC,CAAC;AACrE,OAAO,EAAE,KAAK,YAAY,EAAqC,MAAM,aAAa,CAAC;AAKnF,YAAY,EAAE,YAAY,EAAE,CAAC;AAC7B,MAAM,MAAM,KAAK,GAAG,MAAM,GAAG,IAAI,CAAC;AAClC,MAAM,MAAM,aAAa,GAAG,SAAS,GAAG,KAAK,CAAC;AAuE9C,wBAAgB,iBAAiB,CAAC,MAAM,GAAE,aAAyB,GAAG,MAAM,CAE3E;AAED,wBAAgB,uBAAuB,CAAC,MAAM,GAAE,aAAyB,GAAG,MAAM,CAEjF;AAGD,wBAAsB,cAAc,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAEvF;AAED,wBAAsB,eAAe,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAE1F;AAED,wBAAsB,oBAAoB,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAEpG;AAGD,wBAAsB,cAAc,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAEvF;AAED,wBAAsB,eAAe,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAE1F;AAED,wBAAsB,oBAAoB,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAEpG"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../../src/src/apis/brightdata/llmScraper/index.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mCAAmC,CAAC;AACrE,OAAO,EAAE,KAAK,YAAY,EAAqC,MAAM,aAAa,CAAC;AAKnF,YAAY,EAAE,YAAY,EAAE,CAAC;AAC7B,MAAM,MAAM,KAAK,GAAG,MAAM,GAAG,IAAI,CAAC;AAClC,MAAM,MAAM,aAAa,GAAG,SAAS,GAAG,KAAK,GAAG,SAAS,CAAC;AAiF1D,wBAAgB,iBAAiB,CAAC,MAAM,GAAE,aAAyB,GAAG,MAAM,CAE3E;AAED,wBAAgB,uBAAuB,CAAC,MAAM,GAAE,aAAyB,GAAG,MAAM,CAEjF;AAGD,wBAAsB,cAAc,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAEvF;AAED,wBAAsB,eAAe,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAE1F;AAED,wBAAsB,oBAAoB,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAEpG;AAGD,wBAAsB,cAAc,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAEvF;AAED,wBAAsB,eAAe,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAE1F;AAED,wBAAsB,oBAAoB,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAEpG;AAGD,wBAAsB,iBAAiB,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAEjG"}
package/script/src/apis/brightdata/llmScraper/index.js CHANGED
@@ -41,6 +41,7 @@ exports.downloadGPTSnapshots = downloadGPTSnapshots;
41
41
  exports.scrapeAIMBatch = scrapeAIMBatch;
42
42
  exports.triggerAIMBatch = triggerAIMBatch;
43
43
  exports.downloadAIMSnapshots = downloadAIMSnapshots;
44
+ exports.downloadSnapshots = downloadSnapshots;
44
45
  /* eslint no-console: ["warn", { allow: ["log", "warn", "error"] }] */
45
46
  /**
46
47
  * LLM Scraper - Public API
@@ -100,10 +101,19 @@ function getLLMScraper(target = 'chatgpt') {
100
101
  return existingScraper;
101
102
  }
102
103
  const providerName = getProviderName();
103
- const targetOptions = getTargetOptions(target);
104
- const provider = providerName === 'brightdata'
105
- ? (0, brightdata_js_1.createBrightdataProvider)(targetOptions.brightdata)
106
- : (0, oxy_js_1.createOxylabsProvider)(targetOptions.oxylabs);
104
+ let provider;
105
+ if (target === 'generic') {
106
+ // Generic instance: only used for download/monitor, no target-specific config needed
107
+ provider = providerName === 'brightdata'
108
+ ? (0, brightdata_js_1.createBrightdataProvider)()
109
+ : (0, oxy_js_1.createOxylabsProvider)();
110
+ }
111
+ else {
112
+ const targetOptions = getTargetOptions(target);
113
+ provider = providerName === 'brightdata'
114
+ ? (0, brightdata_js_1.createBrightdataProvider)(targetOptions.brightdata)
115
+ : (0, oxy_js_1.createOxylabsProvider)(targetOptions.oxylabs);
116
+ }
107
117
  const scraper = (0, scrape_js_1.createLLMScraper)(provider);
108
118
  scrapers.set(target, scraper);
109
119
  return scraper;
@@ -126,7 +136,7 @@ async function triggerGPTBatch(options) {
126
136
  return getLLMScraper('chatgpt').triggerLLMBatch(options);
127
137
  }
128
138
  async function downloadGPTSnapshots(jobIds) {
129
- return getLLMScraper('chatgpt').downloadLLMSnapshots(jobIds);
139
+ return downloadSnapshots(jobIds);
130
140
  }
131
141
  // AIM scraper methods
132
142
  async function scrapeAIMBatch(options) {
@@ -136,5 +146,9 @@ async function triggerAIMBatch(options) {
136
146
  return getLLMScraper('aim').triggerLLMBatch(options);
137
147
  }
138
148
  async function downloadAIMSnapshots(jobIds) {
139
- return getLLMScraper('aim').downloadLLMSnapshots(jobIds);
149
+ return downloadSnapshots(jobIds);
150
+ }
151
+ // Generic download — target-agnostic, works with any job IDs
152
+ async function downloadSnapshots(jobIds) {
153
+ return getLLMScraper('generic').downloadLLMSnapshots(jobIds);
140
154
  }
package/script/src/apis/brightdata/llmScraper/oxy.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"oxy.d.ts","sourceRoot":"","sources":["../../../../../src/src/apis/brightdata/llmScraper/oxy.ts"],"names":[],"mappings":"AAeA,OAAO,EAA6C,KAAK,iBAAiB,EAAE,MAAM,aAAa,CAAC;AA2BhG,UAAU,qBAAqB;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC;IAC7B,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,oBAAoB,EAAE,MAAM,CAAC;CAC7B;AAyCD,wBAAgB,qBAAqB,CAAC,SAAS,GAAE,OAAO,CAAC,qBAAqB,CAAM,GAAG,iBAAiB,CAyJvG;AAMD,eAAO,MAAM,eAAe,EAAE,iBAA2C,CAAC"}
1
+ {"version":3,"file":"oxy.d.ts","sourceRoot":"","sources":["../../../../../src/src/apis/brightdata/llmScraper/oxy.ts"],"names":[],"mappings":"AAeA,OAAO,EAA6C,KAAK,iBAAiB,EAAE,MAAM,aAAa,CAAC;AA0BhG,UAAU,qBAAqB;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC;IAC7B,KAAK,EAAE,OAAO,CAAC;IACf,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,oBAAoB,EAAE,MAAM,CAAC;CAC7B;AAyCD,wBAAgB,qBAAqB,CAAC,SAAS,GAAE,OAAO,CAAC,qBAAqB,CAAM,GAAG,iBAAiB,CAyJvG;AAMD,eAAO,MAAM,eAAe,EAAE,iBAA2C,CAAC"}
package/script/src/apis/brightdata/llmScraper/oxy.js CHANGED
@@ -187,8 +187,8 @@ function createOxylabsProvider(overrides = {}) {
187
187
  return {
188
188
  prompt: content.prompt || '',
189
189
  answer: answerText,
190
- answer_text_markdown: answerTextMarkdown,
191
- sources: (0, scrape_js_1.buildSources)(citations),
190
+ answerMarkdown: answerTextMarkdown,
191
+ sources: (0, scrape_js_1.parseSources)(citations),
192
192
  searchQueries: [],
193
193
  };
194
194
  }
package/script/src/apis/brightdata/llmScraper/scrape.d.ts CHANGED
@@ -23,11 +23,10 @@ export interface LLMScraper {
23
23
  }
24
24
  export declare function getAbortSignal(): AbortSignal | undefined;
25
25
  export declare function cleanAnswer(answer: string): string;
26
- export declare function buildSources(citations: Array<{
26
+ export declare function parseSources(citations: Array<{
27
27
  url: string;
28
28
  title?: string;
29
29
  description?: string;
30
- text?: string;
31
30
  cited?: boolean;
32
31
  }>, linksAttached?: Array<{
33
32
  url?: string;
package/script/src/apis/brightdata/llmScraper/scrape.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"scrape.d.ts","sourceRoot":"","sources":["../../../../../src/src/apis/brightdata/llmScraper/scrape.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mCAAmC,CAAC;AACrE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oCAAoC,CAAC;AAOjE,MAAM,WAAW,YAAY;IAC5B,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACvB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/B;AAED,MAAM,WAAW,iBAAiB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,EAAE,MAAM,CAAC;IACvB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,UAAU,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,GAAG,IAAI,KAAK,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAC1G,UAAU,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IAChD,WAAW,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IACjD,iBAAiB,EAAE,CAAC,GAAG,EAAE,OAAO,KAAK,WAAW,GAAG,IAAI,CAAC;CACxD;AAED,MAAM,WAAW,UAAU;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,cAAc,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC;IACvE,eAAe,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC;IAC1E,oBAAoB,EAAE,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC;CACpF;AAMD,wBAAgB,cAAc,IAAI,WAAW,GAAG,SAAS,CAExD;AAED,wBAAgB,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAMlD;AA+BD,wBAAgB,YAAY,CAC3B,SAAS,EAAE,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,OAAO,CAAA;CAAE,CAAC,EACvG,aAAa,GAAE,KAAK,CAAC;IAAE,GAAG,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,CAAM,GAC3E,KAAK,CAAC,MAAM,CAAC,CAoFf;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,YAAY,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,GAAG,WAAW,CAU5G;AAMD,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,iBAAiB,GAAG,UAAU,CAkExE"}
1
+ {"version":3,"file":"scrape.d.ts","sourceRoot":"","sources":["../../../../../src/src/apis/brightdata/llmScraper/scrape.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mCAAmC,CAAC;AACrE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oCAAoC,CAAC;AAOjE,MAAM,WAAW,YAAY;IAC5B,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACvB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/B;AAED,MAAM,WAAW,iBAAiB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,EAAE,MAAM,CAAC;IACvB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,UAAU,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,GAAG,IAAI,KAAK,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;IAC1G,UAAU,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IAChD,WAAW,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IACjD,iBAAiB,EAAE,CAAC,GAAG,EAAE,OAAO,KAAK,WAAW,GAAG,IAAI,CAAC;CACxD;AAED,MAAM,WAAW,UAAU;IAC1B,cAAc,EAAE,MAAM,CAAC;IACvB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,cAAc,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC;IACvE,eAAe,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,OAAO,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC;IAC1E,oBAAoB,EAAE,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC;CACpF;AAMD,wBAAgB,cAAc,IAAI,WAAW,GAAG,SAAS,CAExD;AAED,wBAAgB,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAMlD;AA+BD,wBAAgB,YAAY,CAC3B,SAAS,EAAE,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,OAAO,CAAA;CAAE,CAAC,EACxF,aAAa,GAAE,KAAK,CAAC;IAAE,GAAG,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAAE,CAAM,GAC3E,KAAK,CAAC,MAAM,CAAC,CAyFf;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,YAAY,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,GAAG,WAAW,CAU5G;AAMD,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,iBAAiB,GAAG,UAAU,CAkExE"}
package/script/src/apis/brightdata/llmScraper/scrape.js CHANGED
@@ -35,7 +35,7 @@ var __importStar = (this && this.__importStar) || (function () {
35
35
  Object.defineProperty(exports, "__esModule", { value: true });
36
36
  exports.getAbortSignal = getAbortSignal;
37
37
  exports.cleanAnswer = cleanAnswer;
38
- exports.buildSources = buildSources;
38
+ exports.parseSources = parseSources;
39
39
  exports.emptyModelResult = emptyModelResult;
40
40
  exports.createLLMScraper = createLLMScraper;
41
41
  /* eslint no-console: ["warn", { allow: ["log", "warn", "error"] }] */
@@ -89,7 +89,7 @@ function hasExtraUrlInfo(current, candidate) {
89
89
  return false;
90
90
  }
91
91
  }
92
- function buildSources(citations, linksAttached = []) {
92
+ function parseSources(citations, linksAttached = []) {
93
93
  const sources = [];
94
94
  const sourcesByKey = new Map();
95
95
  const upsertSource = (url, initialTitle, cited) => {
@@ -137,11 +137,15 @@ function buildSources(citations, linksAttached = []) {
137
137
  continue;
138
138
  const key = urlMergeKey(citation.url);
139
139
  const existing = sourcesByKey.get(key);
140
- const title = citation.title || citation.description || citation.text || '';
140
+ const title = citation.title ?? '';
141
+ const snippet = citation.description;
141
142
  if (existing) {
142
143
  if (title) {
143
144
  existing.title = title;
144
145
  }
146
+ if (snippet) {
147
+ existing.snippet = snippet;
148
+ }
145
149
  existing.cited = existing.cited || citation.cited;
146
150
  // Append extra fragment/params from citation
147
151
  if (hasExtraUrlInfo(existing.url, citation.url)) {
@@ -151,6 +155,7 @@ function buildSources(citations, linksAttached = []) {
151
155
  }
152
156
  const source = {
153
157
  title,
158
+ snippet,
154
159
  url: citation.url,
155
160
  domain: (0, urls_js_1.extractDomain)(citation.url),
156
161
  cited: citation.cited,
@@ -174,7 +179,7 @@ function emptyModelResult(providerName, errorMessage, context) {
174
179
  return {
175
180
  prompt: '',
176
181
  answer: '',
177
- answer_text_markdown: '',
182
+ answerMarkdown: '',
178
183
  sources: [],
179
184
  };
180
185
  }
package/script/src/apis/hasdata/helpers.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"helpers.d.ts","sourceRoot":"","sources":["../../../../src/src/apis/hasdata/helpers.ts"],"names":[],"mappings":"AACA,OAAO,EAAe,KAAK,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAEvE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,iCAAiC,CAAC;AAG9D,eAAO,MAAM,mBAAmB,KAAK,CAAC;AAEtC,eAAO,MAAM,oBAAoB,EAAE,WAMlC,CAAC;AAEF,wBAAgB,gBAAgB,IAAI,MAAM,CAMzC;AAED,wBAAsB,qBAAqB,CAC1C,GAAG,EAAE,MAAM,EACX,WAAW,GAAE,WAAkC,GAC7C,OAAO,CAAC,QAAQ,CAAC,CAgCnB;AAED,UAAU,QAAQ;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;CACvB;AAED,UAAU,SAAS;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,uBAAuB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACxC,gBAAgB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACjC,IAAI,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;IACvB,IAAI,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IAC5B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,UAAU,SAAS;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,UAAU;IAC1B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,UAAU,eAAe;IACxB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,MAAM;IACtB,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;CAC9B;AAED,MAAM,WAAW,SAAS;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;CACvB;AAqMD,wBAAgB,QAAQ,CAAC,GAAG,EAAE,UAAU,GAAG,SAAS,CAInD;AAED,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAI/C"}
1
+ {"version":3,"file":"helpers.d.ts","sourceRoot":"","sources":["../../../../src/src/apis/hasdata/helpers.ts"],"names":[],"mappings":"AACA,OAAO,EAAe,KAAK,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAEvE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,iCAAiC,CAAC;AAG9D,eAAO,MAAM,mBAAmB,KAAK,CAAC;AAEtC,eAAO,MAAM,oBAAoB,EAAE,WAMlC,CAAC;AAEF,wBAAgB,gBAAgB,IAAI,MAAM,CAMzC;AAED,wBAAsB,qBAAqB,CAC1C,GAAG,EAAE,MAAM,EACX,WAAW,GAAE,WAAkC,GAC7C,OAAO,CAAC,QAAQ,CAAC,CAgCnB;AAED,UAAU,QAAQ;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;CACvB;AAED,UAAU,SAAS;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,uBAAuB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACxC,gBAAgB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACjC,IAAI,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;IACvB,IAAI,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IAC5B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,UAAU,SAAS;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,UAAU;IAC1B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,UAAU,eAAe;IACxB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,MAAM;IACtB,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;CAC9B;AAED,MAAM,WAAW,SAAS;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;CACvB;AA6MD,wBAAgB,QAAQ,CAAC,GAAG,EAAE,UAAU,GAAG,SAAS,CAInD;AAED,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAI/C"}
package/script/src/apis/hasdata/helpers.js CHANGED
@@ -170,17 +170,25 @@ function parseAIResult(data, { allowNestedOverview = true } = {}) {
170
170
  const refIndexToSourceIndex = new Map();
171
171
  for (const r of refs) {
172
172
  const link = r.link || r.url;
173
- const title = [r.title, r.source, r.snippet].filter(Boolean).join(' - ');
173
+ const title = [r.title, r.source].filter(Boolean).join(' - ');
174
+ const snippet = cleanText(r.snippet || '') || undefined;
174
175
  if (link && r.index != null) {
175
176
  // Deduplicate by URL
176
177
  const existingIdx = sources.findIndex(s => s.url === link);
177
178
  if (existingIdx >= 0) {
178
179
  refIndexToSourceIndex.set(r.index, existingIdx);
180
+ if (!sources[existingIdx].title && title) {
181
+ sources[existingIdx].title = title;
182
+ }
183
+ if (!sources[existingIdx].snippet && snippet) {
184
+ sources[existingIdx].snippet = snippet;
185
+ }
179
186
  }
180
187
  else {
181
188
  refIndexToSourceIndex.set(r.index, sources.length);
182
189
  sources.push({
183
190
  title,
191
+ snippet,
184
192
  url: link,
185
193
  domain: (0, urls_js_1.extractDomain)(link)
186
194
  });
@@ -219,14 +227,16 @@ function parseAIResult(data, { allowNestedOverview = true } = {}) {
219
227
  const sourceIndexes = refIndexes
220
228
  .map(ri => refIndexToSourceIndex.get(ri))
221
229
  .filter((si) => si != null);
222
- for (const si of sourceIndexes) {
230
+ const uniqueSourceIndexes = sourceIndexes.filter((v, i, a) => a.indexOf(v) === i);
231
+ for (const si of uniqueSourceIndexes) {
223
232
  citedSourceIndexes.add(si);
224
233
  sources[si].positions ??= [];
225
- if (!sources[si].positions.includes(parts.length)) {
226
- sources[si].positions.push(parts.length);
234
+ const citationNumber = si + 1;
235
+ if (!sources[si].positions.includes(citationNumber)) {
236
+ sources[si].positions.push(citationNumber);
227
237
  }
228
238
  }
229
- rendered += formatCitationMarkers(sourceIndexes.filter((v, i, a) => a.indexOf(v) === i));
239
+ rendered += formatCitationMarkers(uniqueSourceIndexes);
230
240
  }
231
241
  parts.push(rendered);
232
242
  }
package/script/src/schemas/search.schema.d.ts CHANGED
@@ -4,9 +4,9 @@ export type ContextSize = 'low' | 'medium' | 'high';
4
4
  export type ReasoningEffort = 'low' | 'medium' | 'high';
5
5
  export interface SearchResult {
6
6
  answer: string;
7
- answer_text_markdown?: string;
8
7
  sources: Array<Source>;
9
8
  searchQueries?: Array<string>;
9
+ answerMarkdown?: string;
10
10
  }
11
11
  export type SearchOptions = {
12
12
  prompt: string;
package/script/src/schemas/search.schema.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"search.schema.d.ts","sourceRoot":"","sources":["../../../src/src/schemas/search.schema.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,+CAA+C,CAAC;AAEvE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAClD,MAAM,MAAM,WAAW,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;AACpD,MAAM,MAAM,eAAe,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;AAExD,MAAM,WAAW,YAAY;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACvB,aAAa,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;CAC9B;AAED,MAAM,MAAM,aAAa,GAAG;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;4FACwF;IACxF,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,UAAU,CAAC,EAAE,YAAY,GAAG,oBAAoB,CAAC;CACjD,CAAC;AAEF,MAAM,MAAM,sBAAsB,CAAC,CAAC,IAAI,aAAa,GAAG;IACvD,cAAc,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;CAC7B,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC,GAAG;IAChE,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;CACxB,CAAC"}
1
+ {"version":3,"file":"search.schema.d.ts","sourceRoot":"","sources":["../../../src/src/schemas/search.schema.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,+CAA+C,CAAC;AAEvE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAClD,MAAM,MAAM,WAAW,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;AACpD,MAAM,MAAM,eAAe,GAAG,KAAK,GAAG,QAAQ,GAAG,MAAM,CAAC;AAExD,MAAM,WAAW,YAAY;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACvB,aAAa,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAC9B,cAAc,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,MAAM,aAAa,GAAG;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB;4FACwF;IACxF,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,UAAU,CAAC,EAAE,YAAY,GAAG,oBAAoB,CAAC;CACjD,CAAC;AAEF,MAAM,MAAM,sBAAsB,CAAC,CAAC,IAAI,aAAa,GAAG;IACvD,cAAc,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;CAC7B,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG,IAAI,CAAC,aAAa,EAAE,QAAQ,CAAC,GAAG;IAChE,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;CACxB,CAAC"}