kiku-agent 1.3.0 → 1.3.2

package/server/agent/tests/webExtract.test.js

@@ -0,0 +1,261 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import {
+  decodeHtmlBytes,
+  normalizeCharsetLabel,
+  decodeEntities,
+  normalizeText,
+  extractMainContent,
+  splitIntoPassages,
+  scorePassages,
+  selectPassages,
+  filterUniquePassages,
+  dedupePassages,
+  simhashText,
+  hamming64,
+  tokenizeForMatch,
+  truncateByWords,
+} from '../websearch/extract.js';
+
+const ARTICLE_FIXTURE = `<!DOCTYPE html>
+<html lang="th">
+<head>
+  <title>Ju Jingyi releases new single</title>
+  <meta name="description" content="Coverage of the new single release">
+  <link rel="canonical" href="https://example.com/news/ju-jingyi-single">
+  <script type="application/ld+json">
+  {
+    "@type": "NewsArticle",
+    "headline": "Ju Jingyi releases new single",
+    "datePublished": "2026-09-01",
+    "author": {"name": "News Desk"},
+    "articleBody": "The JSON-LD body of the article."
+  }
+  </script>
+</head>
+<body>
+  <nav class="main-nav"><a href="/home">Home</a><a href="/news">News</a> Navigation bar text</nav>
+  <aside class="sidebar">Sidebar promo banner content here</aside>
+  <div class="related-stories">Related recommended stories widget</div>
+  <article class="post-content">
+    <h1>Ju Jingyi releases new single</h1>
+    <p>Ju Jingyi released her newest single on September 1st, alongside a full music video filmed in Chongqing. The song debuted at the top of several regional charts within two days.</p>
+    <p>Fans praised the choreography and the cinematic production quality of the accompanying video release.</p>
+    <h3>Chart performance</h3>
+    <p>The single topped three regional charts and stayed in the top ten for two consecutive weeks across multiple platforms.</p>
+    <table>
+      <tr><th>Year</th><th>Release</th><th>Label</th></tr>
+      <tr><td>2017</td><td>Don't Fall in Love</td><td>STAR48</td></tr>
+      <tr><td>2024</td><td>0.2s</td><td>Independent</td></tr>
+    </table>
+  </article>
+  <footer class="site-footer">Copyright notice and legal disclaimer footer</footer>
+</body>
+</html>`;
+
+test('decodeHtmlBytes resolves charset from the Content-Type header', () => {
+  const thaiBytes = Uint8Array.from([0xA1, 0xA2, 0xA3]);
+  const decoded = decodeHtmlBytes(thaiBytes, 'text/html; charset=windows-874');
+  assert.equal(decoded.charset, 'windows-874');
+  assert.equal(decoded.text, 'กขฃ');
+});
+
+test('decodeHtmlBytes sniffs legacy meta charset when the header is silent', () => {
+  const html = Buffer.from(
+    '<html><head><meta charset="windows-874"></head><body>\xA1\xA2\xA3</body></html>',
+    'latin1'
+  );
+  const decoded = decodeHtmlBytes(html, 'text/html');
+  assert.equal(decoded.charset, 'windows-874');
+  assert.equal(decoded.text.includes('กขฃ'), true);
+});
+
+test('decodeHtmlBytes decodes GBK content', () => {
+  const gbkBytes = Uint8Array.from([0xD6, 0xD0]);
+  const decoded = decodeHtmlBytes(gbkBytes, 'text/html; charset=gbk');
+  assert.equal(decoded.text, '中');
+});
+
+test('decodeHtmlBytes falls back to utf-8 for unknown or missing charsets', () => {
+  const utf8 = Buffer.from('กขค', 'utf8');
+  const missing = decodeHtmlBytes(utf8, 'text/html');
+  assert.equal(missing.charset, 'utf-8');
+  assert.equal(missing.text, 'กขค');
+  const unknown = decodeHtmlBytes(utf8, 'text/html; charset=made-up-charset');
+  assert.equal(unknown.text, 'กขค');
+});
+
+test('decodeHtmlBytes honors byte order marks over header declarations', () => {
+  const bom = Buffer.concat([Buffer.from([0xEF, 0xBB, 0xBF]), Buffer.from('กขค', 'utf8')]);
+  const decoded = decodeHtmlBytes(bom, 'text/html; charset=windows-874');
+  assert.equal(decoded.charset, 'utf-8');
+  assert.equal(decoded.text, 'กขค');
+});
+
+test('normalizeCharsetLabel maps legacy Thai and Chinese labels', () => {
+  assert.equal(normalizeCharsetLabel('TIS-620'), 'windows-874');
+  assert.equal(normalizeCharsetLabel('gb2312'), 'gbk');
+  assert.equal(normalizeCharsetLabel('ISO-8859-1'), 'windows-1252');
+  assert.equal(normalizeCharsetLabel(null), null);
+});
+
+test('decodeEntities covers numeric, hex and named entities', () => {
+  assert.equal(decodeEntities('&#x2019;'), '’');
+  assert.equal(decodeEntities('&#39;'), "'");
+  assert.equal(decodeEntities('&hellip;'), '...');
+  assert.equal(decodeEntities('&amp;&lt;&gt;'), '&<>');
+  assert.equal(decodeEntities('&unknownentity;'), '&unknownentity;');
+});
+
+test('normalizeText strips zero-width characters and applies NFC', () => {
+  assert.equal(normalizeText('a\u200Bb\u200Dc\uFEFFd'), 'abcd');
+  assert.equal(normalizeText('ก \u00E1  b'), 'ก á b');
+  assert.equal(normalizeText('   spaced\t\ntext   '), 'spaced text');
+});
+
+test('extractMainContent isolates the article body from boilerplate', async () => {
+  const result = await extractMainContent(ARTICLE_FIXTURE, { url: 'https://example.com/news/ju-jingyi-single' });
+
+  assert.equal(result.ok ?? true, true);
+  assert.equal(result.meta.title, 'Ju Jingyi releases new single');
+  assert.equal(result.meta.canonical, 'https://example.com/news/ju-jingyi-single');
+
+  assert.equal(result.markdown.includes('released her newest single'), true);
+  assert.equal(result.markdown.includes('Sidebar promo banner'), false);
+  assert.equal(result.markdown.includes('Navigation bar text'), false);
+  assert.equal(result.markdown.includes('Related recommended stories'), false);
+  assert.equal(result.markdown.includes('Copyright notice and legal disclaimer'), false);
+
+  assert.equal(result.markdown.includes('## Ju Jingyi releases new single'), true);
+  assert.equal(result.markdown.includes('### Chart performance'), true);
+
+  assert.equal(result.markdown.includes('| Year | Release | Label |'), true);
+  assert.equal(result.markdown.includes("| 2017 | Don't Fall in Love | STAR48 |"), true);
+  assert.equal(result.markdown.includes('| 2024 | 0.2s | Independent |'), true);
+
+  assert.equal(result.paragraphs.some(p => p.startsWith('Ju Jingyi released her newest single')), true);
+  assert.equal(result.quality.isThin, false);
+  assert.ok(result.quality.density > 0);
+});
+
+test('extractMainContent parses JSON-LD article metadata', async () => {
+  const result = await extractMainContent(ARTICLE_FIXTURE, { url: 'https://example.com/news/ju-jingyi-single' });
+  assert.ok(result.jsonLd);
+  assert.equal(result.jsonLd.headline, 'Ju Jingyi releases new single');
+  assert.equal(result.jsonLd.datePublished, '2026-09-01');
+  assert.equal(result.jsonLd.author, 'News Desk');
+  assert.equal(result.jsonLd.body, 'The JSON-LD body of the article.');
+});
+
+test('extractMainContent flags thin single-page-app shells', async () => {
+  const spaShell = `<!DOCTYPE html>
+<html><head><title>App</title></head>
+<body style="padding: 40px; background: linear-gradient(#ffffff 0%, #f8f8f8 100%); min-height: 100vh; font-family: sans-serif;">
+  <div id="root"></div>
+  <script src="/static/js/bundle.min.js"></script>
+  ${'<div class="loading-skeleton placeholder shimmer-block"> </div>'.repeat(400)}
+</body></html>`;
+  const result = await extractMainContent(spaShell, { url: 'https://app.example.com/' });
+  assert.equal(result.quality.isThin, true);
+  assert.ok(result.quality.htmlChars > 15_000);
+});
+
+test('splitIntoPassages bounds sections by headings and size', () => {
+  const markdown = [
+    '## Section one',
+    'Body of section one.',
+    '## Section two',
+    'Body of section two.',
+    '## Section three',
+    'Body of section three.',
+  ].join('\n');
+
+  const passages = splitIntoPassages(markdown, { maxChars: 1600 });
+  assert.equal(passages.length, 3);
+  assert.equal(passages[0].heading, '## Section one');
+  assert.equal(passages[1].text.startsWith('## Section two'), true);
+  assert.equal(passages[2].index, 2);
+});
+
+test('selectPassages ranks sections by query relevance and preserves document order', () => {
+  const markdown = [
+    '## Music releases',
+    'The discography lists many albums and singles from past years with varied labels and formats.',
+    '## Cooking recipes',
+    'Boil the noodles for four minutes and mix the sauce gently over low heat before serving.',
+    '## Chart performance',
+    'The 0.2s single topped three regional charts and stayed in the top ten for two weeks.',
+  ].join('\n\n');
+
+  const passages = splitIntoPassages(markdown);
+  const selection = selectPassages(passages, '0.2s single chart performance', { budgetChars: 150 });
+
+  assert.equal(selection.total, 3);
+  assert.ok(selection.selected >= 1);
+  assert.ok(selection.selected < selection.total, 'tight budget must exclude low-relevance sections');
+  assert.equal(selection.text.includes('topped three regional charts'), true);
+  assert.equal(selection.text.includes('Boil the noodles'), false);
+  const chartPos = selection.text.indexOf('Chart performance');
+  const musicPos = selection.text.indexOf('Music releases');
+  if (musicPos !== -1 && chartPos !== -1) {
+    assert.ok(musicPos < chartPos, 'selected passages must follow document order');
+  }
+});
+
+test('scorePassages gives zero to empty queries without crashing', () => {
+  const passages = splitIntoPassages('## A\n\nhello world content here');
+  const scored = scorePassages(passages, '');
+  assert.equal(scored.length, 1);
+  assert.equal(scored[0].score, 0);
+});
+
+test('tokenizeForMatch segments Thai words and uses CJK bigrams', () => {
+  const thaiTokens = tokenizeForMatch('จวีจิ้งอีปล่อยเพลงใหม่', 'th');
+  assert.ok(thaiTokens.length > 1, 'Thai text must be segmented into multiple words');
+  const zhTokens = tokenizeForMatch('鞠婧祎新专辑', 'zh');
+  assert.ok(zhTokens.length >= 3, 'Chinese text must produce character bigrams');
+  const latin = tokenizeForMatch('Ju Jingyi chart', 'en');
+  assert.deepEqual(latin, ['ju', 'jingyi', 'chart']);
+});
+
+test('truncateByWords cuts at word boundaries instead of mid-word', () => {
+  const thaiText = 'จวีจิ้งอีปล่อยเพลงใหม่ล่าสุดพร้อมมิวสิกวิดีโอ '.repeat(20);
+  const truncated = truncateByWords(thaiText, 60, 'th');
+  assert.ok(truncated.length <= 60);
+  assert.ok(thaiText.startsWith(truncated));
+
+  const english = 'word '.repeat(100);
+  const truncatedEnglish = truncateByWords(english, 30, 'en');
+  assert.ok(truncatedEnglish.length <= 30);
+  assert.equal(truncatedEnglish.trimEnd().split(' ').pop(), 'word');
+});
+
+test('simhashText detects near-duplicate passages across pages', () => {
+  const original = 'Ju Jingyi released her newest single on September 1st alongside a full music video filmed in Chongqing.';
+  const paraphrase = 'Ju Jingyi released her newest single on September 1st alongside a full music video filmed in Chongqing!';
+  const different = 'The recipe for the soup requires boiling noodles for exactly four minutes before serving them hot.';
+
+  const state = { hashes: [] };
+  const first = filterUniquePassages([{ index: 0, heading: '', text: original }], state);
+  const duplicate = filterUniquePassages([{ index: 1, heading: '', text: paraphrase }], state);
+  const unique = filterUniquePassages([{ index: 2, heading: '', text: different }], state);
+
+  assert.equal(first.length, 1);
+  assert.equal(duplicate.length, 0, 'near-identical passage must be dropped');
+  assert.equal(unique.length, 1);
+
+  const deduped = dedupePassages([
+    { index: 0, heading: '', text: original },
+    { index: 1, heading: '', text: paraphrase },
+    { index: 2, heading: '', text: different },
+  ]);
+  assert.equal(deduped.length, 2);
+});
+
+test('hamming64 and simhashText behave on identical and disjoint input', () => {
+  const a = simhashText('identical passage text for hashing purposes here');
+  const b = simhashText('identical passage text for hashing purposes here');
+  assert.equal(hamming64(a, b), 0);
+  const c = simhashText('completely unrelated content about chess strategies and endgame play');
+  assert.ok(hamming64(a, c) > 6);
+});

package/server/agent/tests/webSources.test.js

@@ -0,0 +1,247 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+
+import {
+  parseRssFeed,
+  parseBaikePage,
+  parseBaiduResults,
+  detectLanguage,
+  checkUrlLiveness,
+  decodeBaiduUrl,
+} from '../webSearch.js';
+import { executeTool, prepareToolArguments, toolHandlers, toolsSchema } from '../tools.js';
+import { dynamicToolRouter as toolRouter } from '../toolRouter.js';
+import { extractCitations } from '../citations.js';
+import { PUBLIC_REMOTE_TOOL_NAMES } from '../publicTools.js';
+import { getMcpToolNames } from '../../mcp/tools.js';
+
+const NEW_WEB_TOOLS = [
+  'search_news',
+  'fetch_rss',
+  'search_encyclopedia',
+  'search_music_platforms',
+  'search_chinese_web',
+  'search_bilibili',
+  'search_web_images',
+];
+
+test('new web source tools are registered in the schema, router cluster, and public surfaces', () => {
+  const schemaNames = new Set(toolsSchema.map(t => t.function?.name));
+  for (const name of NEW_WEB_TOOLS) {
+    assert.ok(schemaNames.has(name), `toolsSchema should contain ${name}`);
+    assert.ok(toolRouter.toolsByName.has(name), `router should know ${name}`);
+    assert.ok(PUBLIC_REMOTE_TOOL_NAMES.has(name), `web-chat public tools should include ${name}`);
+    assert.ok(typeof toolHandlers[name] === 'function', `toolHandlers should contain ${name}`);
+  }
+  const mcpNames = new Set(getMcpToolNames());
+  for (const name of ['search_news', 'fetch_rss', 'search_encyclopedia', 'search_music_platforms']) {
+    assert.ok(mcpNames.has(name), `MCP should expose ${name} publicly`);
+  }
+});
+
+test('parseRssFeed handles RSS 2.0 items with CDATA and source tags', () => {
+  const xml = `<?xml version="1.0" encoding="UTF-8"?>
+  <rss version="2.0"><channel><title>Feed Title</title>
+    <item>
+      <title><![CDATA[鞠婧祎新剧开播]]></title>
+      <link>https://news.example.com/a</link>
+      <pubDate>Mon, 01 Sep 2026 08:00:00 GMT</pubDate>
+      <description><![CDATA[<p>โพยละครเรื่องใหม่</p>]]></description>
+      <source url="https://ent.example.com">Example Ent</source>
+    </item>
+    <item>
+      <title>Second story</title>
+      <link>https://news.example.com/b</link>
+      <pubDate>Tue, 02 Sep 2026 09:00:00 GMT</pubDate>
+      <description>Plain snippet</description>
+    </item>
+  </channel></rss>`;
+
+  const items = parseRssFeed(xml);
+  assert.equal(items.length, 2);
+  assert.equal(items[0].title, '鞠婧祎新剧开播');
+  assert.equal(items[0].url, 'https://news.example.com/a');
+  assert.equal(items[0].source, 'Example Ent');
+  assert.match(items[0].snippet, /โพยละครเรื่องใหม่/);
+  assert.match(items[0].publishedAt, /Sep 2026/);
+  assert.equal(items[1].source, '');
+});
+
+test('parseRssFeed handles Atom entries with self-closing link tags', () => {
+  const xml = `<?xml version="1.0"?>
+  <feed xmlns="http://www.w3.org/2005/Atom">
+    <title>Atom Feed</title>
+    <entry>
+      <title>  Kiku releases new single </title>
+      <link rel="alternate" href="https://blog.example.com/entry-1"/>
+      <published>2026-08-30T10:00:00Z</published>
+      <summary>Short summary text</summary>
+    </entry>
+  </feed>`;
+
+  const items = parseRssFeed(xml);
+  assert.equal(items.length, 1);
+  assert.equal(items[0].title, 'Kiku releases new single');
+  assert.equal(items[0].url, 'https://blog.example.com/entry-1');
+  assert.equal(items[0].publishedAt, '2026-08-30T10:00:00Z');
+  assert.equal(items[0].snippet, 'Short summary text');
+});
+
+test('parseBaikePage extracts title, summary and paragraphs from a Baike-style page', () => {
+  const html = `<!DOCTYPE html><html><head>
+    <meta name="description" content="鞠婧祎，1994年6月18日出生于四川省遂宁市，中国内地影视女演员、流行乐歌手。">
+    </head><body>
+    <h1>鞠婧祎</h1>
+    <div class="lemma-summary"><div class="para">鞠婧祎1994年6月18日出生于四川省遂宁市。</div>
+    <div class="para">2013年以SNH48成员身份正式出道。</div></div>
+    <p>short</p>
+  </body></html>`;
+
+  const parsed = parseBaikePage(html);
+  assert.equal(parsed.title, '鞠婧祎');
+  assert.match(parsed.summary, /1994年6月18日出生于四川省遂宁市/);
+  assert.ok(parsed.paragraphs.length >= 2);
+  assert.ok(parsed.paragraphs.every(p => p.length > 10));
+});
+
+test('parseBaiduResults extracts title, redirect link and snippet', () => {
+  const html = `<!DOCTYPE html><html><body>
+    <div class="result c-container" srcid="1599">
+      <h3 class="c-title t t tts-title"><a href="http://www.baidu.com/link?url=abc123">鞠婧祎 最新消息 - 新浪娱乐</a></h3>
+      <span class="content-right_8Zs40">新浪娱乐讯 9月1日，鞠婧祎工作室发布最新声明。</span>
+    </div>
+    <div class="result c-container">
+      <h3 class="c-title"><a href="http://www.baidu.com/link?url=def456">第二條結果</a></h3>
+      <span class="content-right_8Zs40">另一段摘要文字，內容超過三十個字符以方便驗證截取邏輯是否正確。</span>
+    </div>
+  </body></html>`;
+
+  const results = parseBaiduResults(html, 8);
+  assert.equal(results.length, 2);
+  assert.equal(results[0].title, '鞠婧祎 最新消息 - 新浪娱乐');
+  assert.equal(results[0].url, 'http://www.baidu.com/link?url=abc123');
+  assert.match(results[0].snippet, /最新声明/);
+  assert.equal(results[0].source, 'Baidu');
+});
+
+test('detectLanguage routes Thai, Chinese, and English terms', () => {
+  assert.equal(detectLanguage('จวีจิ้งอี ล่าสุด'), 'th');
+  assert.equal(detectLanguage('鞠婧祎 最新'), 'zh');
+  assert.equal(detectLanguage('Ju Jingyi latest news'), 'en');
+});
+
+test('search_news requires a query', async () => {
+  const result = await toolHandlers.search_news({});
+  assert.equal(result.status, 'error');
+  assert.match(result.message, /query parameter is required/);
+});
+
+test('fetch_rss requires a valid URL', async () => {
+  const result = await toolHandlers.fetch_rss({ url: 'not-a-url' });
+  assert.equal(result.status, 'error');
+  assert.match(result.message, /HTTP|URL is required|Invalid URL/);
+});
+
+test('search_encyclopedia requires a term and rejects unknown languages', async () => {
+  const missing = await toolHandlers.search_encyclopedia({});
+  assert.equal(missing.status, 'error');
+
+  const badLang = await executeTool('search_encyclopedia', { term: '鞠婧祎', language: 'fr' });
+  assert.equal(badLang.status, 'error');
+  assert.match(badLang.message || JSON.stringify(badLang), /language/i);
+});
+
+test('chinese web tools are gated behind KIKU_ENABLE_CN_WEB_TOOLS', async () => {
+  const previous = process.env.KIKU_ENABLE_CN_WEB_TOOLS;
+  try {
+    delete process.env.KIKU_ENABLE_CN_WEB_TOOLS;
+    for (const [handler, args] of [
+      [toolHandlers.search_chinese_web, { query: '鞠婧祎' }],
+      [toolHandlers.search_bilibili, { keyword: '鞠婧祎' }],
+      [toolHandlers.search_web_images, { query: '鞠婧祎' }],
+    ]) {
+      const gated = await handler(args);
+      assert.equal(gated.status, 'error');
+      assert.equal(gated.gated, true);
+      assert.match(gated.message, /KIKU_ENABLE_CN_WEB_TOOLS/);
+    }
+
+    process.env.KIKU_ENABLE_CN_WEB_TOOLS = 'true';
+    const missingQuery = await toolHandlers.search_chinese_web({});
+    assert.equal(missingQuery.status, 'error');
+    assert.match(missingQuery.message, /query parameter is required/);
+    const missingKeyword = await toolHandlers.search_bilibili({});
+    assert.equal(missingKeyword.status, 'error');
+    assert.match(missingKeyword.message, /keyword parameter is required/);
+    const missingImageQuery = await toolHandlers.search_web_images({});
+    assert.equal(missingImageQuery.status, 'error');
+    assert.match(missingImageQuery.message, /query parameter is required/);
+  } finally {
+    if (previous === undefined) delete process.env.KIKU_ENABLE_CN_WEB_TOOLS;
+    else process.env.KIKU_ENABLE_CN_WEB_TOOLS = previous;
+  }
+});
+
+test('executeTool validates the new tools through the standard pipeline', async () => {
+  const noArgs = await executeTool('search_music_platforms', {});
+  assert.equal(noArgs.status, 'error');
+
+  const prepared = prepareToolArguments('search_news', { query: 'Ju Jingyi', bogus: 1 });
+  assert.equal(prepared.safe, true);
+  assert.deepEqual(prepared.args, { query: 'Ju Jingyi' });
+  assert.match(prepared.warnings.join(' '), /Unknown argument "bogus"/);
+});
+
+test('router exposes news and encyclopedia tools on research-flavored queries', () => {
+  const routedTh = toolRouter.routeToolsForQuery('จวีจิ้งอีมีข่าวอะไรล่าสุดบ้าง');
+  const routedZh = toolRouter.routeToolsForQuery('鞠婧祎 最新新闻动态');
+  for (const routed of [routedTh, routedZh]) {
+    const names = new Set(routed.map(t => t.function?.name));
+    assert.ok(names.has('search_news'), 'search_news should be routed for news queries');
+    assert.ok(names.has('search_encyclopedia'), 'search_encyclopedia should be routed for research queries');
+  }
+});
+
+test('router exposes bilibili and music platform tools on media-flavored queries', () => {
+  const routed = toolRouter.routeToolsForQuery('ฟังเพลงล่าสุดของจวีจิ้งอี และดูคลิป bilibili');
+  const names = new Set(routed.map(t => t.function?.name));
+  assert.ok(names.has('search_bilibili'));
+  assert.ok(names.has('search_music_platforms'));
+});
+
+test('extractCitations works with recorded tool results (recordedTools regression)', () => {
+  const citations = extractCitations(
+    'Ju Jingyi news says a new drama is coming: https://example.com/a',
+    [{ name: 'web_search', result: { results: [{ title: 'Ju Jingyi new drama', url: 'https://example.com/a' }] } }]
+  );
+  assert.equal(citations.length, 1);
+  assert.equal(citations[0].id, 1);
+  assert.equal(citations[0].type, 'web');
+  assert.equal(citations[0].url, 'https://example.com/a');
+});
+
+test('checkUrlLiveness validates safe URLs and catches dead or invalid links', async () => {
+  const invalid = await checkUrlLiveness('not-a-url');
+  assert.equal(invalid.alive, false);
+
+  const empty = await checkUrlLiveness('');
+  assert.equal(empty.alive, false);
+
+  const doc = await checkUrlLiveness('https://example.com/some/article');
+  assert.equal(doc.alive, true);
+
+  const dead = await checkUrlLiveness('https://this-domain-definitely-does-not-exist-at-all-999.xyz/page', { timeoutMs: 500 });
+  assert.equal(dead.alive, false);
+});
+
+test('decodeBaiduUrl decodes Baidu encrypted objURL and fromURL into valid http urls', () => {
+  const encrypted = 'ippr_z2C$qAzdH3FAzdH3Fooo_z&e3Bktstktst_z&e3Bv54AzdH3Fet1j5AzdH3FBV8vv0zm5Ep6';
+  const decoded = decodeBaiduUrl(encrypted);
+  assert.equal(decoded, 'http://www.bilibili.com/video/BV1cc7z6oEtr');
+
+  assert.equal(decodeBaiduUrl('https://example.com/pic.jpg'), 'https://example.com/pic.jpg');
+  assert.equal(decodeBaiduUrl(''), '');
+  assert.equal(decodeBaiduUrl(null), '');
+});
+
+

package/server/agent/websearch/adapters/duckduckgo.js

@@ -0,0 +1,97 @@
+import { createSearchResult } from '../core/types.js';
+import { safeFetchWithPinning } from '../security/ssrf.js';
+import { loadCheerio, decodeEntities, normalizeText } from '../extract.js';
+
+function cleanHtml(raw) {
+  if (!raw || typeof raw !== 'string') return '';
+  return normalizeText(decodeEntities(raw.replace(/<[^>]+>/g, ' ')));
+}
+
+async function parseResultsWithCheerio(html, maxResults) {
+  const cheerioLoad = await loadCheerio();
+  if (!cheerioLoad) return [];
+  const $ = cheerioLoad(html);
+  const results = [];
+  $('div.result, div.web-result').each((_, el) => {
+    if (results.length >= maxResults) return;
+    const $el = $(el);
+    const anchor = $el.find('a.result__a').first();
+    let link = anchor.attr('href') || '';
+    if (link.includes('uddg=')) {
+      try {
+        const decoded = new URL('https://duckduckgo.com' + link).searchParams.get('uddg');
+        if (decoded) link = decoded;
+      } catch {}
+    }
+    const title = normalizeText(decodeEntities(anchor.text()));
+    const snippet = normalizeText(decodeEntities($el.find('.result__snippet').first().text()));
+    if (link && title) results.push({ link, title, snippet });
+  });
+  return results;
+}
+
+function parseResultsWithRegex(html, maxResults) {
+  const results = [];
+  const resultRegex = /<div[^>]*class="[^"]*result[^"]*"[^>]*>[\s\S]*?<a[^>]+class="[^"]*result__a[^"]*"[^>]+href="([^"]+)"[^>]*>([\s\S]*?)<\/a>[\s\S]*?<a[^>]+class="[^"]*result__snippet[^"]*"[^>]*>([\s\S]*?)<\/a>/gi;
+  let match;
+  while ((match = resultRegex.exec(html)) !== null && results.length < maxResults) {
+    results.push({ link: match[1], title: cleanHtml(match[2]), snippet: cleanHtml(match[3]) });
+  }
+  return results;
+}
+
+export async function searchDuckDuckGo(query, options = {}) {
+  const maxResults = Number(options.limit) || 10;
+  const timeoutMs = Number(options.timeoutMs) || 8000;
+  const results = [];
+  const seenLinks = new Set();
+
+  try {
+    const lang = options.language || 'en';
+    const kl = lang === 'th' ? 'th-th' : (lang === 'zh' ? 'zh-cn' : 'en-us');
+    const searchUrl = `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}&kl=${kl}`;
+
+    const res = await safeFetchWithPinning(searchUrl, {
+      headers: {
+        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+        'Accept-Language': 'th,en-US;q=0.9,en;q=0.8,zh-CN;q=0.7'
+      },
+      timeoutMs,
+      signal: options.signal
+    });
+
+    if (!res.ok) {
+      return [];
+    }
+
+    const html = await res.text();
+    const cheerioParsed = await parseResultsWithCheerio(html, maxResults);
+    const parsed = cheerioParsed.length > 0 ? cheerioParsed : parseResultsWithRegex(html, maxResults);
+
+    for (const item of parsed) {
+      if (results.length >= maxResults) break;
+      let rawLink = item.link;
+      if (rawLink.includes('uddg=')) {
+        try {
+          const decoded = new URL('https://duckduckgo.com' + rawLink).searchParams.get('uddg');
+          if (decoded) rawLink = decoded;
+        } catch {}
+      }
+
+      if (rawLink && item.title && !seenLinks.has(rawLink)) {
+        seenLinks.add(rawLink);
+        results.push(createSearchResult({
+          link: rawLink,
+          title: item.title,
+          snippet: item.snippet,
+          engine: 'duckduckgo',
+          score: 1.0 - (results.length * 0.05)
+        }));
+      }
+    }
+  } catch {
+  }
+
+  return results;
+}