@hexiaowu/dsh-web-search-zai 0.0.0-stage → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/parse.js ADDED
@@ -0,0 +1,540 @@
1
+ /**
2
+ * Response parsing for the two Z.ai search channels.
3
+ *
4
+ * The plan channel answers in an Anthropic-compatible envelope, but Zhipu's
5
+ * implementation is not standard: the result block is `tool_result` (not
6
+ * `web_search_tool_result`) and its `content` is a Python-repr string such as
7
+ * `[{'text': [{'title': '…', 'link': 'https://…', 'content': '…'}]}]`.
8
+ * The native API channel answers with `{ code, search_result: [...] }`.
9
+ *
10
+ * Both channels converge on `collectRecords`, which walks a value by its
11
+ * bracket structure — never by scanning for the text "title"/"url" — and treats
12
+ * `link` and `url`, `content` and `snippet`, `page_age` and `publish_date` as
13
+ * equivalent. This module has no host imports, so its fixtures run with plain
14
+ * `node --test`.
15
+ *
16
+ * @module dsh-web-search-zai/parse
17
+ */
18
+ import { dedupeSources, isNonBlankString, isPlainObject, isWebUrl, normalizeUrl } from './util.js';
19
+
20
+ /** Maximum container nesting followed while collecting records. */
21
+ const MAX_PARSE_DEPTH = 16;
22
+
23
+ /** Record key aliases across the standard Anthropic, Zhipu, and native layouts. */
24
+ const URL_KEYS = ['url', 'link'];
25
+ const TITLE_KEYS = ['title', 'name'];
26
+ /**
27
+ * Zhipu's standard-layout `web_search_result` items put the excerpt in
28
+ * `encrypted_content` (the name comes from Anthropic's opaque page blob, but
29
+ * Zhipu fills it with plain text). It is tried after the explicit excerpt
30
+ * fields and before the generic `text` fallback.
31
+ */
32
+ const SNIPPET_KEYS = ['snippet', 'content', 'encrypted_content', 'summary', 'abstract', 'description', 'cited_text', 'text'];
33
+ const DATE_KEYS = ['publishedAt', 'published_at', 'publish_date', 'publishDate', 'page_age', 'pageAge', 'date'];
34
+
35
+ /**
36
+ * Raised when a response cannot yield any source. Carries the stable
37
+ * `WEB_PROVIDER_ERROR` code so the provider can convert it without this module
38
+ * depending on the host package.
39
+ */
40
+ export class SearchParseError extends Error {
41
+ constructor(message) {
42
+ super(message);
43
+ this.name = 'SearchParseError';
44
+ this.code = 'WEB_PROVIDER_ERROR';
45
+ }
46
+ }
47
+
48
+ /**
49
+ * Parse a response body into a search result.
50
+ *
51
+ * @param {unknown} json - the parsed Anthropic-compatible Messages response.
52
+ * @returns {{ sources: object[], truncated: false }}
53
+ * @throws {SearchParseError} when no result block yields a usable URL.
54
+ */
55
+ export function parsePlanResponse(json) {
56
+ if (!isPlainObject(json)) throw new SearchParseError('The Z.ai plan response was not a JSON object.');
57
+ if (json.type === 'error' || isPlainObject(json.error)) {
58
+ throw new SearchParseError(`The Z.ai plan response reported an error: ${describeError(json)}`);
59
+ }
60
+
61
+ const blocks = Array.isArray(json.content) ? json.content : [];
62
+ const records = [];
63
+ for (const block of blocks) {
64
+ if (!isResultBlock(block)) continue;
65
+ collectRecords(block.content === undefined ? block : block.content, records);
66
+ }
67
+ // Unknown block types (or a provider that puts the list somewhere unexpected)
68
+ // still get one structural pass; the fallback only runs when nothing matched.
69
+ if (records.length === 0) collectRecords(blocks, records);
70
+
71
+ const sources = mergeCitations(dedupeSources(records), citationSnippets(blocks));
72
+ if (sources.length === 0) {
73
+ throw new SearchParseError(
74
+ 'The Z.ai plan response contained no web-search result with a usable http(s) URL. '
75
+ + `Content block types: ${describeBlockTypes(blocks)}. `
76
+ + 'The request may not have triggered server-side web_search on this endpoint.',
77
+ );
78
+ }
79
+ return { sources, truncated: false };
80
+ }
81
+
82
+ /**
83
+ * Parse a native `/web_search` response into a search result.
84
+ *
85
+ * @param {unknown} json - the parsed native search response.
86
+ * @returns {{ sources: object[], truncated: false }}
87
+ * @throws {SearchParseError} on a non-200 `code` or no usable result URL.
88
+ */
89
+ export function parseApiResponse(json) {
90
+ if (!isPlainObject(json)) throw new SearchParseError('The Z.ai native search response was not a JSON object.');
91
+
92
+ const { code } = json;
93
+ if (code !== undefined && code !== null && String(code) !== '200') {
94
+ const requestId = nonBlankStringOrUndefined(json.request_id ?? json.requestId);
95
+ throw new SearchParseError(
96
+ `The Z.ai native search API returned code ${String(code)}`
97
+ + `${requestId === undefined ? '' : ` (request_id ${requestId})`}: ${describeError(json)}. `
98
+ + 'Check the search_engine value, the API key permissions, and the account balance.',
99
+ );
100
+ }
101
+
102
+ const payload = json.search_result ?? json.searchResult ?? json.data?.search_result ?? json.data;
103
+ const records = [];
104
+ collectRecords(payload, records);
105
+ if (records.length === 0) collectRecords(json, records);
106
+
107
+ const sources = dedupeSources(records);
108
+ if (sources.length === 0) {
109
+ throw new SearchParseError(
110
+ 'The Z.ai native search response contained no result with a usable http(s) URL. '
111
+ + `Top-level keys: ${Object.keys(json).join(', ') || 'none'}.`,
112
+ );
113
+ }
114
+ return { sources, truncated: false };
115
+ }
116
+
117
+ /**
118
+ * Walk an arbitrary nested value (object, array, JSON string, or Python-repr
119
+ * string) and collect every record that names a valid web URL.
120
+ *
121
+ * @param {unknown} value - the value to walk.
122
+ * @returns {object[]} deduplicated sources in first-seen order.
123
+ */
124
+ export function parseSearchPayload(value) {
125
+ const records = [];
126
+ collectRecords(value, records);
127
+ return dedupeSources(records);
128
+ }
129
+
130
+ /**
131
+ * Parse a JSON string, falling back to a Python-literal parser. Returns
132
+ * `undefined` when the text is not a structured value.
133
+ *
134
+ * @param {string} text - candidate JSON or Python repr.
135
+ * @returns {unknown} the parsed value, or `undefined`.
136
+ */
137
+ export function parseStructuredText(text) {
138
+ if (typeof text !== 'string') return undefined;
139
+ const trimmed = text.trim();
140
+ if (trimmed.length === 0) return undefined;
141
+
142
+ const first = trimmed[0];
143
+ if (first === '"' || first === "'") {
144
+ // A repr of a string that itself contains the list (double encoding).
145
+ const decoded = decodeQuotedText(trimmed, first);
146
+ if (typeof decoded !== 'string') return undefined;
147
+ const inner = decoded.trim();
148
+ if (inner.startsWith('[') || inner.startsWith('{')) return parseStructuredText(inner);
149
+ return undefined;
150
+ }
151
+ if (first !== '[' && first !== '{' && first !== '(') return undefined;
152
+
153
+ try {
154
+ return JSON.parse(trimmed);
155
+ } catch {
156
+ // Fall through to the Python-literal grammar.
157
+ }
158
+ try {
159
+ return new PythonLiteralParser(trimmed).parseSequence();
160
+ } catch {
161
+ return undefined;
162
+ }
163
+ }
164
+
165
+ function decodeQuotedText(text, quote) {
166
+ if (quote === '"') {
167
+ try {
168
+ return JSON.parse(text);
169
+ } catch {
170
+ // Fall through to the Python grammar for JSON-invalid escapes.
171
+ }
172
+ }
173
+ try {
174
+ return new PythonLiteralParser(text).parseSequence();
175
+ } catch {
176
+ return undefined;
177
+ }
178
+ }
179
+
180
+ /** Whether a content block is a search-result carrier in either layout. */
181
+ function isResultBlock(block) {
182
+ if (!isPlainObject(block)) return false;
183
+ return block.type === 'web_search_tool_result'
184
+ || block.type === 'tool_result'
185
+ || block.type === 'web_search_result'
186
+ || block.type === 'search_result';
187
+ }
188
+
189
+ /** Collect records from a value by its container structure. */
190
+ function collectRecords(value, out, depth = 0) {
191
+ if (depth > MAX_PARSE_DEPTH) return;
192
+ if (typeof value === 'string') {
193
+ const parsed = parseStructuredText(value);
194
+ if (parsed !== undefined) collectRecords(parsed, out, depth + 1);
195
+ return;
196
+ }
197
+ if (Array.isArray(value)) {
198
+ for (const item of value) collectRecords(item, out, depth + 1);
199
+ return;
200
+ }
201
+ if (!isPlainObject(value)) return;
202
+
203
+ const source = toSource(value);
204
+ if (source !== undefined) {
205
+ out.push(source);
206
+ return;
207
+ }
208
+ for (const child of Object.values(value)) collectRecords(child, out, depth + 1);
209
+ }
210
+
211
+ /** Project one record object to a seam source, or `undefined` when it is not one. */
212
+ function toSource(record) {
213
+ if (hasText(record.cited_text)) return undefined; // a citation, not a result item
214
+ const url = firstString(record, URL_KEYS);
215
+ if (!isWebUrl(url)) return undefined;
216
+
217
+ const title = firstString(record, TITLE_KEYS);
218
+ const snippet = firstReadableSnippet(record, SNIPPET_KEYS);
219
+ const publishedAt = firstString(record, DATE_KEYS);
220
+ return {
221
+ url: url.trim(),
222
+ ...(title === undefined ? {} : { title }),
223
+ ...(snippet === undefined ? {} : { snippet, summary: snippet }),
224
+ ...(publishedAt === undefined ? {} : { publishedAt }),
225
+ };
226
+ }
227
+
228
+ /** Build a `normalized URL → cited_text` map from every `text` block's citations. */
229
+ function citationSnippets(blocks) {
230
+ const snippets = new Map();
231
+ for (const block of Array.isArray(blocks) ? blocks : []) {
232
+ if (!isPlainObject(block) || block.type !== 'text' || !Array.isArray(block.citations)) continue;
233
+ for (const citation of block.citations) {
234
+ const url = firstString(citation, URL_KEYS);
235
+ const text = firstString(citation, ['cited_text', 'snippet', 'text']);
236
+ if (url === undefined || text === undefined) continue;
237
+ const key = normalizeUrl(url);
238
+ if (key !== undefined && !snippets.has(key)) snippets.set(key, text);
239
+ }
240
+ }
241
+ return snippets;
242
+ }
243
+
244
+ /** Fill a missing snippet from the matching citation (first occurrence wins). */
245
+ function mergeCitations(sources, snippets) {
246
+ if (snippets.size === 0) return sources;
247
+ return sources.map((source) => {
248
+ if (source.snippet !== undefined) return source;
249
+ const excerpt = snippets.get(normalizeUrl(source.url));
250
+ return excerpt === undefined ? source : { ...source, snippet: excerpt, summary: excerpt };
251
+ });
252
+ }
253
+
254
+ function firstString(record, keys) {
255
+ if (!isPlainObject(record)) return undefined;
256
+ for (const key of keys) {
257
+ const value = record[key];
258
+ if (isNonBlankString(value)) return value.trim();
259
+ }
260
+ return undefined;
261
+ }
262
+
263
+ /**
264
+ * Pick the first excerpt that reads like prose. Anthropic-compatible providers
265
+ * may also put an opaque base64 blob in `encrypted_content`; that must not
266
+ * become the model-facing summary, so a long whitespace-free base64-shaped
267
+ * value is skipped and the next key is tried.
268
+ */
269
+ function firstReadableSnippet(record, keys) {
270
+ if (!isPlainObject(record)) return undefined;
271
+ for (const key of keys) {
272
+ const value = record[key];
273
+ if (!isNonBlankString(value)) continue;
274
+ const trimmed = value.trim();
275
+ if (looksOpaque(trimmed)) continue;
276
+ return trimmed;
277
+ }
278
+ return undefined;
279
+ }
280
+
281
+ function looksOpaque(value) {
282
+ return value.length > 120 && !/\s/u.test(value) && /^[A-Za-z0-9+/=_-]+$/u.test(value);
283
+ }
284
+
285
+ function hasText(value) {
286
+ return isNonBlankString(value);
287
+ }
288
+
289
+ function nonBlankStringOrUndefined(value) {
290
+ return isNonBlankString(value) ? value.trim() : undefined;
291
+ }
292
+
293
+ function describeBlockTypes(blocks) {
294
+ if (blocks.length === 0) return 'none';
295
+ return blocks.map((block) => (typeof block?.type === 'string' ? block.type : typeof block)).join(', ');
296
+ }
297
+
298
+ /** Extract the most specific human-readable error detail from a response. */
299
+ function describeError(value) {
300
+ if (typeof value === 'string') return value.trim();
301
+ if (!isPlainObject(value)) return String(value ?? 'unknown error');
302
+ for (const key of ['error', 'message', 'msg', 'detail', 'error_message']) {
303
+ const detail = value[key];
304
+ if (typeof detail === 'string' && detail.trim().length > 0) return detail.trim();
305
+ if (isPlainObject(detail)) {
306
+ const nested = describeError(detail);
307
+ if (nested.length > 0 && nested !== 'unknown error') return nested;
308
+ }
309
+ }
310
+ try {
311
+ return JSON.stringify(value).slice(0, 300);
312
+ } catch {
313
+ return 'unknown error';
314
+ }
315
+ }
316
+
317
+ /**
318
+ * Recursive-descent parser for Python literal reprs. Supports lists, tuples,
319
+ * dicts, single- and double-quoted strings, integers/floats/nan/inf,
320
+ * `True`/`False`/`None`, and the escapes Zhipu actually emits (`\uXXXX`,
321
+ * `\xXX`, `\\`, quotes, and control escapes). Record boundaries come from the
322
+ * bracket nesting, so a comma or brace inside a title cannot split a record.
323
+ */
324
+ class PythonLiteralParser {
325
+ constructor(text) {
326
+ this.text = text;
327
+ this.index = 0;
328
+ }
329
+
330
+ parseSequence() {
331
+ this.skipWhitespace();
332
+ const values = [this.parseValue()];
333
+ this.skipWhitespace();
334
+ while (this.peek() === ',') {
335
+ this.index += 1;
336
+ this.skipWhitespace();
337
+ if (this.index >= this.text.length) break;
338
+ values.push(this.parseValue());
339
+ this.skipWhitespace();
340
+ }
341
+ if (this.index !== this.text.length) this.fail('unexpected trailing input');
342
+ return values.length === 1 ? values[0] : values;
343
+ }
344
+
345
+ parseValue() {
346
+ this.skipWhitespace();
347
+ const char = this.peek();
348
+ if (char === undefined) this.fail('unexpected end of input');
349
+ if (char === '[') return this.parseSequenceContainer(']');
350
+ if (char === '(') return this.parseSequenceContainer(')');
351
+ if (char === '{') return this.parseObject();
352
+ if (char === '"' || char === "'") return this.parseString(char);
353
+
354
+ const rest = this.text.slice(this.index);
355
+ const keyword = /^(?:True|true)\b/.exec(rest);
356
+ if (keyword !== null) {
357
+ this.index += keyword[0].length;
358
+ return true;
359
+ }
360
+ const falsy = /^(?:False|false)\b/.exec(rest);
361
+ if (falsy !== null) {
362
+ this.index += falsy[0].length;
363
+ return false;
364
+ }
365
+ const none = /^(?:None|null|undefined)\b/.exec(rest);
366
+ if (none !== null) {
367
+ this.index += none[0].length;
368
+ return null;
369
+ }
370
+ const number = /^[+-]?(?:inf|infinity|nan|(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?)/i.exec(rest);
371
+ if (number !== null) {
372
+ this.index += number[0].length;
373
+ return toNumber(number[0]);
374
+ }
375
+ this.fail('unsupported value');
376
+ }
377
+
378
+ parseSequenceContainer(close) {
379
+ this.index += 1;
380
+ const values = [];
381
+ this.skipWhitespace();
382
+ if (this.peek() === close) {
383
+ this.index += 1;
384
+ return values;
385
+ }
386
+ for (;;) {
387
+ values.push(this.parseValue());
388
+ this.skipWhitespace();
389
+ const char = this.peek();
390
+ if (char === ',') {
391
+ this.index += 1;
392
+ this.skipWhitespace();
393
+ if (this.peek() === close) {
394
+ this.index += 1;
395
+ return values;
396
+ }
397
+ continue;
398
+ }
399
+ if (char === close) {
400
+ this.index += 1;
401
+ return values;
402
+ }
403
+ this.fail(`expected "," or "${close}"`);
404
+ }
405
+ }
406
+
407
+ parseObject() {
408
+ this.index += 1;
409
+ const object = {};
410
+ this.skipWhitespace();
411
+ if (this.peek() === '}') {
412
+ this.index += 1;
413
+ return object;
414
+ }
415
+ for (;;) {
416
+ const key = this.parseKey();
417
+ this.skipWhitespace();
418
+ if (this.peek() !== ':') this.fail('expected ":"');
419
+ this.index += 1;
420
+ object[key] = this.parseValue();
421
+ this.skipWhitespace();
422
+ const char = this.peek();
423
+ if (char === ',') {
424
+ this.index += 1;
425
+ this.skipWhitespace();
426
+ if (this.peek() === '}') {
427
+ this.index += 1;
428
+ return object;
429
+ }
430
+ continue;
431
+ }
432
+ if (char === '}') {
433
+ this.index += 1;
434
+ return object;
435
+ }
436
+ this.fail('expected "," or "}"');
437
+ }
438
+ }
439
+
440
+ parseKey() {
441
+ this.skipWhitespace();
442
+ const char = this.peek();
443
+ if (char === '"' || char === "'") return this.parseString(char);
444
+ const identifier = /^[A-Za-z_][A-Za-z0-9_]*/.exec(this.text.slice(this.index));
445
+ if (identifier !== null) {
446
+ this.index += identifier[0].length;
447
+ return identifier[0];
448
+ }
449
+ const number = /^[+-]?\d+/.exec(this.text.slice(this.index));
450
+ if (number !== null) {
451
+ this.index += number[0].length;
452
+ return number[0];
453
+ }
454
+ this.fail('expected an object key');
455
+ }
456
+
457
+ parseString(quote) {
458
+ this.index += 1;
459
+ let out = '';
460
+ for (;;) {
461
+ if (this.index >= this.text.length) this.fail('unterminated string');
462
+ const char = this.text[this.index];
463
+ this.index += 1;
464
+ if (char === quote) return out;
465
+ if (char !== '\\') {
466
+ out += char;
467
+ continue;
468
+ }
469
+ if (this.index >= this.text.length) this.fail('unterminated escape');
470
+ const escape = this.text[this.index];
471
+ this.index += 1;
472
+ switch (escape) {
473
+ case 'n': out += '\n'; break;
474
+ case 'r': out += '\r'; break;
475
+ case 't': out += '\t'; break;
476
+ case 'b': out += '\b'; break;
477
+ case 'f': out += '\f'; break;
478
+ case 'v': out += '\v'; break;
479
+ case 'a': out += '\u0007'; break;
480
+ case '0': case '1': case '2': case '3':
481
+ case '4': case '5': case '6': case '7': {
482
+ let digits = escape;
483
+ while (digits.length < 3 && /[0-7]/u.test(this.peek() ?? '')) {
484
+ digits += this.text[this.index];
485
+ this.index += 1;
486
+ }
487
+ out += String.fromCharCode(Number.parseInt(digits, 8));
488
+ break;
489
+ }
490
+ case 'x': out += String.fromCharCode(this.readHex(2)); break;
491
+ case 'u': out += this.readUnicodeEscape(); break;
492
+ case 'U': out += String.fromCodePoint(this.readHex(8)); break;
493
+ case '\n': break;
494
+ case '\\': out += '\\'; break;
495
+ case '"': out += '"'; break;
496
+ case "'": out += "'"; break;
497
+ case '/': out += '/'; break;
498
+ default: out += `\\${escape}`; break;
499
+ }
500
+ }
501
+ }
502
+
503
+ readUnicodeEscape() {
504
+ const code = this.readHex(4);
505
+ if (code >= 0xd800 && code <= 0xdbff && this.text.startsWith('\\u', this.index)) {
506
+ const low = Number.parseInt(this.text.slice(this.index + 2, this.index + 6), 16);
507
+ if (Number.isInteger(low) && low >= 0xdc00 && low <= 0xdfff) {
508
+ this.index += 6;
509
+ return String.fromCodePoint(0x10000 + ((code - 0xd800) << 10) + (low - 0xdc00));
510
+ }
511
+ }
512
+ return String.fromCharCode(code);
513
+ }
514
+
515
+ readHex(count) {
516
+ const raw = this.text.slice(this.index, this.index + count);
517
+ if (raw.length !== count || !/^[0-9a-fA-F]+$/u.test(raw)) this.fail(`expected ${count} hex digits`);
518
+ this.index += count;
519
+ return Number.parseInt(raw, 16);
520
+ }
521
+
522
+ peek() {
523
+ return this.text[this.index];
524
+ }
525
+
526
+ skipWhitespace() {
527
+ while (this.index < this.text.length && /\s/u.test(this.text[this.index])) this.index += 1;
528
+ }
529
+
530
+ fail(reason) {
531
+ throw new SyntaxError(`${reason} at offset ${this.index}`);
532
+ }
533
+ }
534
+
535
+ function toNumber(raw) {
536
+ const lower = raw.toLowerCase();
537
+ if (lower.endsWith('inf') || lower.endsWith('infinity')) return lower.startsWith('-') ? -Infinity : Infinity;
538
+ if (lower.endsWith('nan')) return Number.NaN;
539
+ return Number(raw);
540
+ }