@graphext/cuery 0.10.3 → 0.10.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (9) hide show
  1. package/esm/src/apis/brightdata/llmScraper/scrape.js +1 -1
  2. package/esm/src/apis/hasdata/helpers.d.ts +1 -0
  3. package/esm/src/apis/hasdata/helpers.d.ts.map +1 -1
  4. package/esm/src/apis/hasdata/helpers.js +72 -15
  5. package/package.json +1 -1
  6. package/script/src/apis/brightdata/llmScraper/scrape.js +1 -1
  7. package/script/src/apis/hasdata/helpers.d.ts +1 -0
  8. package/script/src/apis/hasdata/helpers.d.ts.map +1 -1
  9. package/script/src/apis/hasdata/helpers.js +72 -15
package/esm/src/apis/brightdata/llmScraper/scrape.js CHANGED
@@ -84,7 +84,7 @@ export function parseSources(citations, linksAttached = []) {
84
84
  for (const link of sortedLinks) {
85
85
  if (!link.url)
86
86
  continue;
87
- const source = upsertSource(link.url, link.text ?? '', true);
87
+ const source = upsertSource(link.url, '', true);
88
88
  if (link.position != null) {
89
89
  source.positions ??= [];
90
90
  if (!source.positions.includes(link.position)) {
package/esm/src/apis/hasdata/helpers.d.ts CHANGED
@@ -47,6 +47,7 @@ export interface AIMode {
47
47
  }
48
48
  export interface AIOParsed {
49
49
  answer: string;
50
+ answerMarkdown?: string;
50
51
  sources: Array<Source>;
51
52
  }
52
53
  export declare function parseAIO(aio: AIOverview): AIOParsed;
package/esm/src/apis/hasdata/helpers.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"helpers.d.ts","sourceRoot":"","sources":["../../../../src/src/apis/hasdata/helpers.ts"],"names":[],"mappings":"AACA,OAAO,EAAe,KAAK,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAEvE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,iCAAiC,CAAC;AAG9D,eAAO,MAAM,mBAAmB,KAAK,CAAC;AAEtC,eAAO,MAAM,oBAAoB,EAAE,WAMlC,CAAC;AAEF,wBAAgB,gBAAgB,IAAI,MAAM,CAMzC;AAED,wBAAsB,qBAAqB,CAC1C,GAAG,EAAE,MAAM,EACX,WAAW,GAAE,WAAkC,GAC7C,OAAO,CAAC,QAAQ,CAAC,CAgCnB;AAED,UAAU,QAAQ;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;CACvB;AAED,UAAU,SAAS;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,uBAAuB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACxC,gBAAgB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACjC,IAAI,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;IACvB,IAAI,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IAC5B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,UAAU,SAAS;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,UAAU;IAC1B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,UAAU,eAAe;IACxB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,MAAM;IACtB,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;CAC9B;AAED,MAAM,WAAW,SAAS;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;CACvB;AA6MD,wBAAgB,QAAQ,CAAC,GAAG,EAAE,UAAU,GAAG,SAAS,CAInD;AAED,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAI/C"}
1
+ {"version":3,"file":"helpers.d.ts","sourceRoot":"","sources":["../../../../src/src/apis/hasdata/helpers.ts"],"names":[],"mappings":"AACA,OAAO,EAAe,KAAK,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAEvE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,iCAAiC,CAAC;AAG9D,eAAO,MAAM,mBAAmB,KAAK,CAAC;AAEtC,eAAO,MAAM,oBAAoB,EAAE,WAMlC,CAAC;AAEF,wBAAgB,gBAAgB,IAAI,MAAM,CAMzC;AAED,wBAAsB,qBAAqB,CAC1C,GAAG,EAAE,MAAM,EACX,WAAW,GAAE,WAAkC,GAC7C,OAAO,CAAC,QAAQ,CAAC,CAgCnB;AAED,UAAU,QAAQ;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;CACvB;AAED,UAAU,SAAS;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,uBAAuB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACxC,gBAAgB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACjC,IAAI,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;IACvB,IAAI,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IAC5B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,UAAU,SAAS;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,UAAU;IAC1B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,UAAU,eAAe;IACxB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,MAAM;IACtB,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;CAC9B;AAED,MAAM,WAAW,SAAS;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;CACvB;AA0QD,wBAAgB,QAAQ,CAAC,GAAG,EAAE,UAAU,GAAG,SAAS,CAInD;AAED,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAI/C"}
package/esm/src/apis/hasdata/helpers.js CHANGED
@@ -92,6 +92,19 @@ function* iterListItems(items, indent = 0) {
92
92
  }
93
93
  }
94
94
  }
95
+ function* iterPlainListItems(items) {
96
+ for (const obj of items) {
97
+ const title = obj.title || '';
98
+ const snippet = obj.snippet || '';
99
+ const line = [title, snippet].filter(Boolean).join(' ').trim();
100
+ if (line) {
101
+ yield cleanText(line);
102
+ }
103
+ if (obj.list && Array.isArray(obj.list)) {
104
+ yield* iterPlainListItems(obj.list);
105
+ }
106
+ }
107
+ }
95
108
  function formatTable(block) {
96
109
  const rows = block.rows || [];
97
110
  if (rows.length === 0) {
@@ -116,6 +129,18 @@ function formatCode(block) {
116
129
  const header = `[Code${lang ? ': ' + lang : ''}]`;
117
130
  return `${header}\n${snippet.trim()}`;
118
131
  }
132
+ function formatPlainTable(block) {
133
+ const rows = block.rows || [];
134
+ if (rows.length === 0) {
135
+ return '';
136
+ }
137
+ return rows
138
+ .map((row) => row.map(cell => removeCSSChunks(cell)).join(' | '))
139
+ .join('\n');
140
+ }
141
+ function formatPlainCode(block) {
142
+ return cleanText(block.snippet || '');
143
+ }
119
144
  function formatCitationMarkers(refIndexes) {
120
145
  if (refIndexes.length === 0) {
121
146
  return '';
@@ -130,7 +155,7 @@ function parseAIResult(data, { allowNestedOverview = true } = {}) {
130
155
  const refIndexToSourceIndex = new Map();
131
156
  for (const r of refs) {
132
157
  const link = r.link || r.url;
133
- const title = [r.title, r.source].filter(Boolean).join(' - ');
158
+ const title = r.title ?? '';
134
159
  const snippet = cleanText(r.snippet || '') || undefined;
135
160
  if (link && r.index != null) {
136
161
  // Deduplicate by URL
@@ -156,8 +181,15 @@ function parseAIResult(data, { allowNestedOverview = true } = {}) {
156
181
  }
157
182
  }
158
183
  const citedSourceIndexes = new Set();
159
- const parts = [];
160
- const handlers = {
184
+ const answerParts = [];
185
+ const answerMarkdownParts = [];
186
+ const plainHandlers = {
187
+ paragraph: (b) => cleanText(b.snippet || ''),
188
+ list: (b) => Array.from(iterPlainListItems(b.list || [])).join('\n'),
189
+ table: formatPlainTable,
190
+ code: formatPlainCode
191
+ };
192
+ const markdownHandlers = {
161
193
  paragraph: (b) => cleanText(b.snippet || ''),
162
194
  list: (b) => Array.from(iterListItems(b.list || [])).join('\n'),
163
195
  table: formatTable,
@@ -168,10 +200,12 @@ function parseAIResult(data, { allowNestedOverview = true } = {}) {
168
200
  if (!btype || btype === 'carousel') {
169
201
  continue;
170
202
  }
171
- const handler = handlers[btype];
203
+ const plainHandler = plainHandlers[btype];
204
+ const markdownHandler = markdownHandlers[btype];
172
205
  let rendered = '';
173
- if (handler) {
174
- rendered = handler(block);
206
+ let renderedMarkdown = '';
207
+ if (plainHandler) {
208
+ rendered = plainHandler(block);
175
209
  }
176
210
  else {
177
211
  const snippet = block.snippet || '';
@@ -179,7 +213,16 @@ function parseAIResult(data, { allowNestedOverview = true } = {}) {
179
213
  rendered = cleanText(snippet);
180
214
  }
181
215
  }
182
- if (rendered) {
216
+ if (markdownHandler) {
217
+ renderedMarkdown = markdownHandler(block);
218
+ }
219
+ else {
220
+ const snippet = block.snippet || '';
221
+ if (snippet) {
222
+ renderedMarkdown = cleanText(snippet);
223
+ }
224
+ }
225
+ if (rendered || renderedMarkdown) {
183
226
  // Append citation markers and track positions
184
227
  const refIndexes = block.referenceIndexes || [];
185
228
  if (refIndexes.length > 0) {
@@ -196,27 +239,41 @@ function parseAIResult(data, { allowNestedOverview = true } = {}) {
196
239
  sources[si].positions.push(citationNumber);
197
240
  }
198
241
  }
199
- rendered += formatCitationMarkers(uniqueSourceIndexes);
242
+ const citationMarkers = formatCitationMarkers(uniqueSourceIndexes);
243
+ rendered += citationMarkers;
244
+ renderedMarkdown += citationMarkers;
200
245
  }
201
- parts.push(rendered);
246
+ answerParts.push(rendered);
247
+ answerMarkdownParts.push(renderedMarkdown);
202
248
  }
203
249
  }
204
250
  // Mark cited sources
205
251
  for (const si of citedSourceIndexes) {
206
252
  sources[si].cited = true;
207
253
  }
208
- const deduped = [];
209
- for (const p of parts) {
210
- if (deduped.length === 0 || deduped[deduped.length - 1] !== p) {
211
- deduped.push(p);
254
+ const dedupedAnswer = [];
255
+ for (const part of answerParts) {
256
+ if (dedupedAnswer.length === 0 || dedupedAnswer[dedupedAnswer.length - 1] !== part) {
257
+ dedupedAnswer.push(part);
212
258
  }
213
259
  }
214
- let answer = cleanText(deduped.join('\n\n'));
260
+ const dedupedAnswerMarkdown = [];
261
+ for (const part of answerMarkdownParts) {
262
+ if (dedupedAnswerMarkdown.length === 0 || dedupedAnswerMarkdown[dedupedAnswerMarkdown.length - 1] !== part) {
263
+ dedupedAnswerMarkdown.push(part);
264
+ }
265
+ }
266
+ let answer = cleanText(dedupedAnswer.join('\n\n'));
267
+ let answerMarkdown = cleanText(dedupedAnswerMarkdown.join('\n\n'));
215
268
  if (answer.length > 16000) {
216
269
  console.warn('Warning: AI answer truncated to 16000 characters');
217
270
  answer = answer.slice(0, 16000);
218
271
  }
219
- return { answer, sources };
272
+ if (answerMarkdown.length > 16000) {
273
+ console.warn('Warning: AI markdown answer truncated to 16000 characters');
274
+ answerMarkdown = answerMarkdown.slice(0, 16000);
275
+ }
276
+ return { answer, answerMarkdown, sources };
220
277
  }
221
278
  export function parseAIO(aio) {
222
279
  return parseAIResult(aio, {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@graphext/cuery",
3
- "version": "0.10.3",
3
+ "version": "0.10.5",
4
4
  "description": "Cuery tools for AI-powered keyword research and brand analysis",
5
5
  "repository": {
6
6
  "type": "git",
package/script/src/apis/brightdata/llmScraper/scrape.js CHANGED
@@ -124,7 +124,7 @@ function parseSources(citations, linksAttached = []) {
124
124
  for (const link of sortedLinks) {
125
125
  if (!link.url)
126
126
  continue;
127
- const source = upsertSource(link.url, link.text ?? '', true);
127
+ const source = upsertSource(link.url, '', true);
128
128
  if (link.position != null) {
129
129
  source.positions ??= [];
130
130
  if (!source.positions.includes(link.position)) {
package/script/src/apis/hasdata/helpers.d.ts CHANGED
@@ -47,6 +47,7 @@ export interface AIMode {
47
47
  }
48
48
  export interface AIOParsed {
49
49
  answer: string;
50
+ answerMarkdown?: string;
50
51
  sources: Array<Source>;
51
52
  }
52
53
  export declare function parseAIO(aio: AIOverview): AIOParsed;
package/script/src/apis/hasdata/helpers.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"helpers.d.ts","sourceRoot":"","sources":["../../../../src/src/apis/hasdata/helpers.ts"],"names":[],"mappings":"AACA,OAAO,EAAe,KAAK,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAEvE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,iCAAiC,CAAC;AAG9D,eAAO,MAAM,mBAAmB,KAAK,CAAC;AAEtC,eAAO,MAAM,oBAAoB,EAAE,WAMlC,CAAC;AAEF,wBAAgB,gBAAgB,IAAI,MAAM,CAMzC;AAED,wBAAsB,qBAAqB,CAC1C,GAAG,EAAE,MAAM,EACX,WAAW,GAAE,WAAkC,GAC7C,OAAO,CAAC,QAAQ,CAAC,CAgCnB;AAED,UAAU,QAAQ;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;CACvB;AAED,UAAU,SAAS;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,uBAAuB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACxC,gBAAgB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACjC,IAAI,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;IACvB,IAAI,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IAC5B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,UAAU,SAAS;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,UAAU;IAC1B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,UAAU,eAAe;IACxB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,MAAM;IACtB,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;CAC9B;AAED,MAAM,WAAW,SAAS;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;CACvB;AA6MD,wBAAgB,QAAQ,CAAC,GAAG,EAAE,UAAU,GAAG,SAAS,CAInD;AAED,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAI/C"}
1
+ {"version":3,"file":"helpers.d.ts","sourceRoot":"","sources":["../../../../src/src/apis/hasdata/helpers.ts"],"names":[],"mappings":"AACA,OAAO,EAAe,KAAK,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAEvE,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,iCAAiC,CAAC;AAG9D,eAAO,MAAM,mBAAmB,KAAK,CAAC;AAEtC,eAAO,MAAM,oBAAoB,EAAE,WAMlC,CAAC;AAEF,wBAAgB,gBAAgB,IAAI,MAAM,CAMzC;AAED,wBAAsB,qBAAqB,CAC1C,GAAG,EAAE,MAAM,EACX,WAAW,GAAE,WAAkC,GAC7C,OAAO,CAAC,QAAQ,CAAC,CAgCnB;AAED,UAAU,QAAQ;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;CACvB;AAED,UAAU,SAAS;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,uBAAuB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACxC,gBAAgB,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACjC,IAAI,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;IACvB,IAAI,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IAC5B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,UAAU,SAAS;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,UAAU;IAC1B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,UAAU,eAAe;IACxB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,GAAG,CAAC,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,MAAM;IACtB,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;IAC9B,UAAU,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;CAC9B;AAED,MAAM,WAAW,SAAS;IACzB,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;CACvB;AA0QD,wBAAgB,QAAQ,CAAC,GAAG,EAAE,UAAU,GAAG,SAAS,CAInD;AAED,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAI/C"}
package/script/src/apis/hasdata/helpers.js CHANGED
@@ -132,6 +132,19 @@ function* iterListItems(items, indent = 0) {
132
132
  }
133
133
  }
134
134
  }
135
+ function* iterPlainListItems(items) {
136
+ for (const obj of items) {
137
+ const title = obj.title || '';
138
+ const snippet = obj.snippet || '';
139
+ const line = [title, snippet].filter(Boolean).join(' ').trim();
140
+ if (line) {
141
+ yield cleanText(line);
142
+ }
143
+ if (obj.list && Array.isArray(obj.list)) {
144
+ yield* iterPlainListItems(obj.list);
145
+ }
146
+ }
147
+ }
135
148
  function formatTable(block) {
136
149
  const rows = block.rows || [];
137
150
  if (rows.length === 0) {
@@ -156,6 +169,18 @@ function formatCode(block) {
156
169
  const header = `[Code${lang ? ': ' + lang : ''}]`;
157
170
  return `${header}\n${snippet.trim()}`;
158
171
  }
172
+ function formatPlainTable(block) {
173
+ const rows = block.rows || [];
174
+ if (rows.length === 0) {
175
+ return '';
176
+ }
177
+ return rows
178
+ .map((row) => row.map(cell => removeCSSChunks(cell)).join(' | '))
179
+ .join('\n');
180
+ }
181
+ function formatPlainCode(block) {
182
+ return cleanText(block.snippet || '');
183
+ }
159
184
  function formatCitationMarkers(refIndexes) {
160
185
  if (refIndexes.length === 0) {
161
186
  return '';
@@ -170,7 +195,7 @@ function parseAIResult(data, { allowNestedOverview = true } = {}) {
170
195
  const refIndexToSourceIndex = new Map();
171
196
  for (const r of refs) {
172
197
  const link = r.link || r.url;
173
- const title = [r.title, r.source].filter(Boolean).join(' - ');
198
+ const title = r.title ?? '';
174
199
  const snippet = cleanText(r.snippet || '') || undefined;
175
200
  if (link && r.index != null) {
176
201
  // Deduplicate by URL
@@ -196,8 +221,15 @@ function parseAIResult(data, { allowNestedOverview = true } = {}) {
196
221
  }
197
222
  }
198
223
  const citedSourceIndexes = new Set();
199
- const parts = [];
200
- const handlers = {
224
+ const answerParts = [];
225
+ const answerMarkdownParts = [];
226
+ const plainHandlers = {
227
+ paragraph: (b) => cleanText(b.snippet || ''),
228
+ list: (b) => Array.from(iterPlainListItems(b.list || [])).join('\n'),
229
+ table: formatPlainTable,
230
+ code: formatPlainCode
231
+ };
232
+ const markdownHandlers = {
201
233
  paragraph: (b) => cleanText(b.snippet || ''),
202
234
  list: (b) => Array.from(iterListItems(b.list || [])).join('\n'),
203
235
  table: formatTable,
@@ -208,10 +240,12 @@ function parseAIResult(data, { allowNestedOverview = true } = {}) {
208
240
  if (!btype || btype === 'carousel') {
209
241
  continue;
210
242
  }
211
- const handler = handlers[btype];
243
+ const plainHandler = plainHandlers[btype];
244
+ const markdownHandler = markdownHandlers[btype];
212
245
  let rendered = '';
213
- if (handler) {
214
- rendered = handler(block);
246
+ let renderedMarkdown = '';
247
+ if (plainHandler) {
248
+ rendered = plainHandler(block);
215
249
  }
216
250
  else {
217
251
  const snippet = block.snippet || '';
@@ -219,7 +253,16 @@ function parseAIResult(data, { allowNestedOverview = true } = {}) {
219
253
  rendered = cleanText(snippet);
220
254
  }
221
255
  }
222
- if (rendered) {
256
+ if (markdownHandler) {
257
+ renderedMarkdown = markdownHandler(block);
258
+ }
259
+ else {
260
+ const snippet = block.snippet || '';
261
+ if (snippet) {
262
+ renderedMarkdown = cleanText(snippet);
263
+ }
264
+ }
265
+ if (rendered || renderedMarkdown) {
223
266
  // Append citation markers and track positions
224
267
  const refIndexes = block.referenceIndexes || [];
225
268
  if (refIndexes.length > 0) {
@@ -236,27 +279,41 @@ function parseAIResult(data, { allowNestedOverview = true } = {}) {
236
279
  sources[si].positions.push(citationNumber);
237
280
  }
238
281
  }
239
- rendered += formatCitationMarkers(uniqueSourceIndexes);
282
+ const citationMarkers = formatCitationMarkers(uniqueSourceIndexes);
283
+ rendered += citationMarkers;
284
+ renderedMarkdown += citationMarkers;
240
285
  }
241
- parts.push(rendered);
286
+ answerParts.push(rendered);
287
+ answerMarkdownParts.push(renderedMarkdown);
242
288
  }
243
289
  }
244
290
  // Mark cited sources
245
291
  for (const si of citedSourceIndexes) {
246
292
  sources[si].cited = true;
247
293
  }
248
- const deduped = [];
249
- for (const p of parts) {
250
- if (deduped.length === 0 || deduped[deduped.length - 1] !== p) {
251
- deduped.push(p);
294
+ const dedupedAnswer = [];
295
+ for (const part of answerParts) {
296
+ if (dedupedAnswer.length === 0 || dedupedAnswer[dedupedAnswer.length - 1] !== part) {
297
+ dedupedAnswer.push(part);
252
298
  }
253
299
  }
254
- let answer = cleanText(deduped.join('\n\n'));
300
+ const dedupedAnswerMarkdown = [];
301
+ for (const part of answerMarkdownParts) {
302
+ if (dedupedAnswerMarkdown.length === 0 || dedupedAnswerMarkdown[dedupedAnswerMarkdown.length - 1] !== part) {
303
+ dedupedAnswerMarkdown.push(part);
304
+ }
305
+ }
306
+ let answer = cleanText(dedupedAnswer.join('\n\n'));
307
+ let answerMarkdown = cleanText(dedupedAnswerMarkdown.join('\n\n'));
255
308
  if (answer.length > 16000) {
256
309
  console.warn('Warning: AI answer truncated to 16000 characters');
257
310
  answer = answer.slice(0, 16000);
258
311
  }
259
- return { answer, sources };
312
+ if (answerMarkdown.length > 16000) {
313
+ console.warn('Warning: AI markdown answer truncated to 16000 characters');
314
+ answerMarkdown = answerMarkdown.slice(0, 16000);
315
+ }
316
+ return { answer, answerMarkdown, sources };
260
317
  }
261
318
  function parseAIO(aio) {
262
319
  return parseAIResult(aio, {