kiku-agent 1.3.1 → 1.3.3

package/server/agent/tests/webChatMemory.test.js

@@ -0,0 +1,112 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import os from 'node:os';
+import path from 'node:path';
+import { mkdtemp, readFile, rm } from 'node:fs/promises';
+
+import { KikuAgent } from '../agentCore.js';
+import { McpClientManager } from '../mcpClient.js';
+import { AgentSession } from '../session.js';
+import { AgentMemory } from '../memory.js';
+
+function createMemoryStub() {
+  return {
+    _loaded: true,
+    _storageError: null,
+    facts: [],
+    preferences: { userSkills: [], skillsEnabled: true, topicsOfInterest: [] },
+    buildMemoryContext: () => '',
+    buildUserSkillContext: () => '',
+    trackTopicInterest: () => {},
+    getArchivedSessions: () => [],
+    getStatus: () => ({ factsCount: 0, sessionsCount: 0 }),
+    addFact: () => null,
+  };
+}
+
+test('AgentSession keeps web chat storage files isolated', async () => {
+  const tempDir = await mkdtemp(path.join(os.tmpdir(), 'kiku-web-sessions-'));
+  const firstPath = path.join(tempDir, 'first.json');
+  const secondPath = path.join(tempDir, 'second.json');
+
+  try {
+    const first = new AgentSession({ sessionId: 'web_first', storageFile: firstPath });
+    first.createUserMessage('ข้อความของห้องแรก');
+    await first.saveToDisk();
+
+    const second = new AgentSession({ sessionId: 'web_second', storageFile: secondPath });
+    second.createUserMessage('ข้อความของห้องที่สอง');
+    await second.saveToDisk();
+
+    const loadedFirst = new AgentSession({ storageFile: firstPath });
+    const loadedSecond = new AgentSession({ storageFile: secondPath });
+    assert.equal(await loadedFirst.loadFromDisk(), true);
+    assert.equal(await loadedSecond.loadFromDisk(), true);
+    assert.equal(loadedFirst.messages[0].parts[0].content, 'ข้อความของห้องแรก');
+    assert.equal(loadedSecond.messages[0].parts[0].content, 'ข้อความของห้องที่สอง');
+  } finally {
+    await rm(tempDir, { recursive: true, force: true });
+  }
+});
+
+test('AgentMemory reports fact counts by category', () => {
+  const memory = new AgentMemory();
+  memory.facts = [
+    { category: 'music' },
+    { category: 'music' },
+    { category: 'drama' },
+    {},
+  ];
+
+  assert.deepEqual(memory.getFactsByCategory(), { music: 2, drama: 1, general: 1 });
+  assert.deepEqual(memory.getStatus().factsByCategory, { music: 2, drama: 1, general: 1 });
+});
+
+test('KikuAgent hydrates browser history and persists direct answers', async () => {
+  const tempDir = await mkdtemp(path.join(os.tmpdir(), 'kiku-web-memory-'));
+  const storageFile = path.join(tempDir, 'session.json');
+  const observedMessages = [];
+  const client = {
+    provider: 'lmstudio',
+    providerName: 'LM Studio',
+    model: 'test-model',
+    async chat(options) {
+      observedMessages.push(options.messages.map(message => ({ ...message })));
+      options.onToken?.('ฉันจำได้ว่าชื่อที่บอกคือ Miju');
+      return { role: 'assistant', content: 'ฉันจำได้ว่าชื่อที่บอกคือ Miju' };
+    }
+  };
+  const agent = new KikuAgent({
+    client,
+    memory: createMemoryStub(),
+    mcpManager: new McpClientManager({
+      autoLoad: false,
+      globalConfigPath: path.join(tempDir, 'mcp.json'),
+    }),
+    sessionId: 'web_memory_test',
+    sessionStorageFile: storageFile,
+  });
+
+  try {
+    await agent.ready();
+    const hydration = await agent.hydrateConversationHistory([
+      { role: 'user', text: 'เรียกฉันว่าอะไรดี' },
+      { role: 'bot', text: 'ฉันจะเรียกคุณว่า Miju' },
+    ], { sessionId: 'web_memory_test' });
+    assert.equal(hydration.hydrated, true);
+
+    const answer = await agent.run('ฉันบอกให้เรียกว่าอะไร');
+    assert.equal(answer, 'ฉันจำได้ว่าชื่อที่บอกคือ Miju');
+    assert.ok(observedMessages[0].some(message =>
+      message.role === 'assistant' && message.content === 'ฉันจะเรียกคุณว่า Miju'
+    ));
+
+    const saved = JSON.parse(await readFile(storageFile, 'utf8'));
+    assert.equal(saved.id, 'web_memory_test');
+    assert.equal(saved.messages.length, 4);
+    assert.equal(saved.messages.at(-1).parts[0].content, 'ฉันจำได้ว่าชื่อที่บอกคือ Miju');
+  } finally {
+    await agent.mcpManager.disconnectAll();
+    await rm(tempDir, { recursive: true, force: true });
+  }
+});

package/server/agent/tests/webChatSecurity.test.js

@@ -0,0 +1,169 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import http from 'node:http';
+import { readFileSync } from 'node:fs';
+import {
+  resolveAllowedOrigin,
+  applySecurityHeaders,
+  extractApiKey,
+  checkChatRateLimit,
+  resolveClientIp
+} from '../../web-chat/core/security.js';
+import { handleWebChatRequest } from '../../web-chat/router.js';
+import { getSharedAgent } from '../../routes/agent.js';
+
+test('Security: resolveAllowedOrigin correctly validates allowed origins', () => {
+  assert.equal(resolveAllowedOrigin('http://localhost:3000'), 'http://localhost:3000');
+  assert.equal(resolveAllowedOrigin('http://127.0.0.1:5173'), 'http://127.0.0.1:5173');
+  assert.equal(resolveAllowedOrigin('https://jujingyi-fans.vercel.app'), null);
+  assert.equal(resolveAllowedOrigin(null), null);
+  assert.equal(resolveAllowedOrigin(''), null);
+});
+
+test('Security: resolveAllowedOrigin respects custom WEBCHAT_CORS_ORIGINS', () => {
+  const origEnv = process.env.WEBCHAT_CORS_ORIGINS;
+  try {
+    process.env.WEBCHAT_CORS_ORIGINS = 'https://portal.kiku.com, https://admin.kiku.com';
+    assert.equal(resolveAllowedOrigin('https://portal.kiku.com'), 'https://portal.kiku.com');
+    assert.equal(resolveAllowedOrigin('https://admin.kiku.com'), 'https://admin.kiku.com');
+    assert.equal(resolveAllowedOrigin('https://evil-site.com'), null);
+  } finally {
+    if (origEnv !== undefined) process.env.WEBCHAT_CORS_ORIGINS = origEnv;
+    else delete process.env.WEBCHAT_CORS_ORIGINS;
+  }
+});
+
+test('Security: applySecurityHeaders injects standard defensive headers', () => {
+  const headers = {};
+  const mockReq = { headers: { origin: 'http://localhost:3000' } };
+  const mockRes = {
+    setHeader: (k, v) => { headers[k.toLowerCase()] = v; }
+  };
+
+  applySecurityHeaders(mockReq, mockRes);
+  assert.equal(headers['x-content-type-options'], 'nosniff');
+  assert.equal(headers['x-frame-options'], 'SAMEORIGIN');
+  assert.equal(headers['referrer-policy'], 'strict-origin-when-cross-origin');
+  assert.equal(headers['access-control-allow-origin'], 'http://localhost:3000');
+  assert.match(headers['access-control-allow-methods'], /GET, POST, OPTIONS/);
+  assert.match(headers['access-control-allow-headers'], /Authorization/);
+});
+
+test('Security: extractApiKey extracts tokens securely with priority', () => {
+  
+  const req1 = { headers: { 'x-api-key': 'key_from_x_header' } };
+  assert.equal(extractApiKey(req1), 'key_from_x_header');
+
+
+  const req2 = { headers: { authorization: 'Bearer key_from_bearer' } };
+  assert.equal(extractApiKey(req2), 'key_from_bearer');
+
+
+  const req3 = { headers: { 'x-api-key': 'key_header' } };
+  const url3 = new URL('http://localhost:3000/api/chat?apiKey=key_query');
+  assert.equal(extractApiKey(req3, url3), 'key_header');
+
+  
+  const req4 = { headers: {} };
+  const url4 = new URL('http://localhost:3000/api/chat?apiKey=key_query');
+  assert.equal(extractApiKey(req4, url4), null);
+
+  
+  assert.equal(extractApiKey({ headers: {} }, new URL('http://localhost:3000/api/chat')), null);
+});
+
+test('Security: client IP ignores spoofed forwarding headers unless a trusted proxy is configured', () => {
+  const original = process.env.KIKU_TRUST_PROXY;
+  const originalVercel = process.env.VERCEL;
+  try {
+    delete process.env.KIKU_TRUST_PROXY;
+    delete process.env.VERCEL;
+    const req = { headers: { 'x-forwarded-for': '10.0.0.5' }, socket: { remoteAddress: '198.51.100.20' } };
+    assert.equal(resolveClientIp(req), '198.51.100.20');
+  } finally {
+    if (original === undefined) delete process.env.KIKU_TRUST_PROXY;
+    else process.env.KIKU_TRUST_PROXY = original;
+    if (originalVercel === undefined) delete process.env.VERCEL;
+    else process.env.VERCEL = originalVercel;
+  }
+});
+
+test('Security: remotely reachable agents are isolated and cannot access privileged tools', () => {
+  const first = getSharedAgent();
+  const second = getSharedAgent();
+  assert.notEqual(first, second);
+  assert.notEqual(first.memory, second.memory);
+  const names = new Set(first.getAllToolsSchema().map(tool => tool.function?.name || tool.name));
+  assert.equal(names.has('read_local_file'), false);
+  assert.equal(names.has('manage_skill'), false);
+  assert.equal(names.has('fetch_json'), false);
+});
+
+test('Security: web-chat escapes markdown before rendering and never persists API keys in browser storage', () => {
+  const source = readFileSync(new URL('../../web-chat/ui/scripts/client.js', import.meta.url), 'utf8');
+  assert.match(source, /text = escapeHtml\(reconstructed\)/);
+  assert.match(source, /data-query=.*this\.dataset\.query/);
+  assert.doesNotMatch(source, /localStorage\.setItem\([^\n]*(?:api.?key|vip.?key|token|secret)/i);
+  assert.doesNotMatch(source, /body\.apiKey/);
+});
+
+test('Security: checkChatRateLimit enforces 60 requests per minute', () => {
+  const testIp = '198.51.100.42';
+  const first = checkChatRateLimit(testIp);
+  assert.equal(first.allowed, true);
+  assert.equal(typeof first.remaining, 'number');
+
+
+  for (let i = 0; i < 60; i++) {
+    checkChatRateLimit(testIp);
+  }
+
+  const blocked = checkChatRateLimit(testIp);
+  assert.equal(blocked.allowed, false);
+  assert.ok(blocked.retryAfter > 0);
+
+  
+  const otherIpResult = checkChatRateLimit('203.0.113.99');
+  assert.equal(otherIpResult.allowed, true);
+});
+
+test('Integration: web-chat router handles OPTIONS preflight and rate limiting over HTTP', async () => {
+  const server = http.createServer(async (req, res) => {
+    await handleWebChatRequest(req, res, { port: 0 });
+  });
+
+  await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve));
+  const port = server.address().port;
+  const baseUrl = `http://127.0.0.1:${port}`;
+
+  try {
+  
+    const optRes = await fetch(`${baseUrl}/api/chat`, {
+      method: 'OPTIONS',
+      headers: { Origin: 'http://localhost:3000' }
+    });
+    assert.equal(optRes.status, 204);
+    assert.equal(optRes.headers.get('access-control-allow-origin'), 'http://localhost:3000');
+    assert.equal(optRes.headers.get('x-content-type-options'), 'nosniff');
+
+
+    const badReq = await fetch(`${baseUrl}/api/chat`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({})
+    });
+    assert.equal(badReq.status, 400);
+    const badJson = await badReq.json();
+    assert.equal(badJson.error, 'Query is required');
+
+    
+    const statusRes = await fetch(`${baseUrl}/api/status`, {
+      headers: { Authorization: 'Bearer test_agent_key_123' }
+    });
+    assert.equal(statusRes.status, 200);
+    const statusJson = await statusRes.json();
+    assert.equal(statusJson.status, 'ok');
+  } finally {
+    await new Promise((resolve) => server.close(resolve));
+  }
+});

package/server/agent/tests/webExtract.test.js

@@ -0,0 +1,284 @@
+import test from 'node:test';
+import assert from 'node:assert/strict';
+import {
+  decodeHtmlBytes,
+  normalizeCharsetLabel,
+  decodeEntities,
+  normalizeText,
+  extractMainContent,
+  splitIntoPassages,
+  scorePassages,
+  selectPassages,
+  filterUniquePassages,
+  dedupePassages,
+  simhashText,
+  hamming64,
+  tokenizeForMatch,
+  truncateByWords,
+} from '../websearch/extract.js';
+import { buildPageResult } from '../webSearch.js';
+
+const ARTICLE_FIXTURE = `<!DOCTYPE html>
+<html lang="th">
+<head>
+  <title>Ju Jingyi releases new single</title>
+  <meta name="description" content="Coverage of the new single release">
+  <link rel="canonical" href="https://example.com/news/ju-jingyi-single">
+  <script type="application/ld+json">
+  {
+    "@type": "NewsArticle",
+    "headline": "Ju Jingyi releases new single",
+    "datePublished": "2026-09-01",
+    "author": {"name": "News Desk"},
+    "articleBody": "The JSON-LD body of the article."
+  }
+  </script>
+</head>
+<body>
+  <nav class="main-nav"><a href="/home">Home</a><a href="/news">News</a> Navigation bar text</nav>
+  <aside class="sidebar">Sidebar promo banner content here</aside>
+  <div class="related-stories">Related recommended stories widget</div>
+  <article class="post-content">
+    <h1>Ju Jingyi releases new single</h1>
+    <p>Ju Jingyi released her newest single on September 1st, alongside a full music video filmed in Chongqing. The song debuted at the top of several regional charts within two days.</p>
+    <p>Fans praised the choreography and the cinematic production quality of the accompanying video release.</p>
+    <h3>Chart performance</h3>
+    <p>The single topped three regional charts and stayed in the top ten for two consecutive weeks across multiple platforms.</p>
+    <table>
+      <tr><th>Year</th><th>Release</th><th>Label</th></tr>
+      <tr><td>2017</td><td>Don't Fall in Love</td><td>STAR48</td></tr>
+      <tr><td>2024</td><td>0.2s</td><td>Independent</td></tr>
+    </table>
+  </article>
+  <footer class="site-footer">Copyright notice and legal disclaimer footer</footer>
+</body>
+</html>`;
+
+test('decodeHtmlBytes resolves charset from the Content-Type header', () => {
+  const thaiBytes = Uint8Array.from([0xA1, 0xA2, 0xA3]);
+  const decoded = decodeHtmlBytes(thaiBytes, 'text/html; charset=windows-874');
+  assert.equal(decoded.charset, 'windows-874');
+  assert.equal(decoded.text, 'กขฃ');
+});
+
+test('decodeHtmlBytes sniffs legacy meta charset when the header is silent', () => {
+  const html = Buffer.from(
+    '<html><head><meta charset="windows-874"></head><body>\xA1\xA2\xA3</body></html>',
+    'latin1'
+  );
+  const decoded = decodeHtmlBytes(html, 'text/html');
+  assert.equal(decoded.charset, 'windows-874');
+  assert.equal(decoded.text.includes('กขฃ'), true);
+});
+
+test('decodeHtmlBytes decodes GBK content', () => {
+  const gbkBytes = Uint8Array.from([0xD6, 0xD0]);
+  const decoded = decodeHtmlBytes(gbkBytes, 'text/html; charset=gbk');
+  assert.equal(decoded.text, '中');
+});
+
+test('decodeHtmlBytes falls back to utf-8 for unknown or missing charsets', () => {
+  const utf8 = Buffer.from('กขค', 'utf8');
+  const missing = decodeHtmlBytes(utf8, 'text/html');
+  assert.equal(missing.charset, 'utf-8');
+  assert.equal(missing.text, 'กขค');
+  const unknown = decodeHtmlBytes(utf8, 'text/html; charset=made-up-charset');
+  assert.equal(unknown.text, 'กขค');
+});
+
+test('decodeHtmlBytes honors byte order marks over header declarations', () => {
+  const bom = Buffer.concat([Buffer.from([0xEF, 0xBB, 0xBF]), Buffer.from('กขค', 'utf8')]);
+  const decoded = decodeHtmlBytes(bom, 'text/html; charset=windows-874');
+  assert.equal(decoded.charset, 'utf-8');
+  assert.equal(decoded.text, 'กขค');
+});
+
+test('normalizeCharsetLabel maps legacy Thai and Chinese labels', () => {
+  assert.equal(normalizeCharsetLabel('TIS-620'), 'windows-874');
+  assert.equal(normalizeCharsetLabel('gb2312'), 'gbk');
+  assert.equal(normalizeCharsetLabel('ISO-8859-1'), 'windows-1252');
+  assert.equal(normalizeCharsetLabel(null), null);
+});
+
+test('decodeEntities covers numeric, hex and named entities', () => {
+  assert.equal(decodeEntities('&#x2019;'), '’');
+  assert.equal(decodeEntities('&#39;'), "'");
+  assert.equal(decodeEntities('&hellip;'), '...');
+  assert.equal(decodeEntities('&amp;&lt;&gt;'), '&<>');
+  assert.equal(decodeEntities('&unknownentity;'), '&unknownentity;');
+});
+
+test('normalizeText strips zero-width characters and applies NFC', () => {
+  assert.equal(normalizeText('a\u200Bb\u200Dc\uFEFFd'), 'abcd');
+  assert.equal(normalizeText('ก \u00E1  b'), 'ก á b');
+  assert.equal(normalizeText('   spaced\t\ntext   '), 'spaced text');
+});
+
+test('extractMainContent isolates the article body from boilerplate', async () => {
+  const result = await extractMainContent(ARTICLE_FIXTURE, { url: 'https://example.com/news/ju-jingyi-single' });
+
+  assert.equal(result.ok ?? true, true);
+  assert.equal(result.meta.title, 'Ju Jingyi releases new single');
+  assert.equal(result.meta.canonical, 'https://example.com/news/ju-jingyi-single');
+
+  assert.equal(result.markdown.includes('released her newest single'), true);
+  assert.equal(result.markdown.includes('Sidebar promo banner'), false);
+  assert.equal(result.markdown.includes('Navigation bar text'), false);
+  assert.equal(result.markdown.includes('Related recommended stories'), false);
+  assert.equal(result.markdown.includes('Copyright notice and legal disclaimer'), false);
+
+  assert.equal(result.markdown.includes('## Ju Jingyi releases new single'), true);
+  assert.equal(result.markdown.includes('### Chart performance'), true);
+
+  assert.equal(result.markdown.includes('| Year | Release | Label |'), true);
+  assert.equal(result.markdown.includes("| 2017 | Don't Fall in Love | STAR48 |"), true);
+  assert.equal(result.markdown.includes('| 2024 | 0.2s | Independent |'), true);
+
+  assert.equal(result.paragraphs.some(p => p.startsWith('Ju Jingyi released her newest single')), true);
+  assert.equal(result.quality.isThin, false);
+  assert.ok(result.quality.density > 0);
+});
+
+test('extractMainContent parses JSON-LD article metadata', async () => {
+  const result = await extractMainContent(ARTICLE_FIXTURE, { url: 'https://example.com/news/ju-jingyi-single' });
+  assert.ok(result.jsonLd);
+  assert.equal(result.jsonLd.headline, 'Ju Jingyi releases new single');
+  assert.equal(result.jsonLd.datePublished, '2026-09-01');
+  assert.equal(result.jsonLd.author, 'News Desk');
+  assert.equal(result.jsonLd.body, 'The JSON-LD body of the article.');
+});
+
+test('extractMainContent flags thin single-page-app shells', async () => {
+  const spaShell = `<!DOCTYPE html>
+<html><head><title>App</title></head>
+<body style="padding: 40px; background: linear-gradient(#ffffff 0%, #f8f8f8 100%); min-height: 100vh; font-family: sans-serif;">
+  <div id="root"></div>
+  <script src="/static/js/bundle.min.js"></script>
+  ${'<div class="loading-skeleton placeholder shimmer-block"> </div>'.repeat(400)}
+</body></html>`;
+  const result = await extractMainContent(spaShell, { url: 'https://app.example.com/' });
+  assert.equal(result.quality.isThin, true);
+  assert.ok(result.quality.htmlChars > 15_000);
+});
+
+test('buildPageResult reports login-walled/JS-shell pages as actionable errors, not hollow success', () => {
+  const empty = buildPageResult(
+    { markdown: '   ', meta: {}, engine: 'cheerio-readability', quality: { isThin: true, textChars: 0 } },
+    { finalUrl: 'https://weibo.com/u/3669102477', maxLength: 8000, query: '', cached: false }
+  );
+  assert.equal(empty.status, 'error');
+  assert.equal(empty.contentLength, 0);
+  assert.equal(empty.extraction.mode, 'no_content');
+  assert.match(empty.message, /m\.weibo\.cn/);
+  assert.match(empty.message, /search_chinese_web/);
+  assert.match(empty.message, /search_news/);
+  assert.match(empty.message, /Do NOT present this as the final answer/);
+  assert.ok(!('content' in empty) || empty.content === undefined, 'no dead-end sentence for the model to parrot');
+
+  const ok = buildPageResult(
+    { markdown: 'Hello world', meta: {}, engine: 'cheerio-readability', quality: { isThin: false } },
+    { finalUrl: 'https://example.com/a', maxLength: 8000, query: '', cached: false }
+  );
+  assert.equal(ok.status, 'success');
+  assert.equal(ok.content, 'Hello world');
+});
+
+test('splitIntoPassages bounds sections by headings and size', () => {
+  const markdown = [
+    '## Section one',
+    'Body of section one.',
+    '## Section two',
+    'Body of section two.',
+    '## Section three',
+    'Body of section three.',
+  ].join('\n');
+
+  const passages = splitIntoPassages(markdown, { maxChars: 1600 });
+  assert.equal(passages.length, 3);
+  assert.equal(passages[0].heading, '## Section one');
+  assert.equal(passages[1].text.startsWith('## Section two'), true);
+  assert.equal(passages[2].index, 2);
+});
+
+test('selectPassages ranks sections by query relevance and preserves document order', () => {
+  const markdown = [
+    '## Music releases',
+    'The discography lists many albums and singles from past years with varied labels and formats.',
+    '## Cooking recipes',
+    'Boil the noodles for four minutes and mix the sauce gently over low heat before serving.',
+    '## Chart performance',
+    'The 0.2s single topped three regional charts and stayed in the top ten for two weeks.',
+  ].join('\n\n');
+
+  const passages = splitIntoPassages(markdown);
+  const selection = selectPassages(passages, '0.2s single chart performance', { budgetChars: 150 });
+
+  assert.equal(selection.total, 3);
+  assert.ok(selection.selected >= 1);
+  assert.ok(selection.selected < selection.total, 'tight budget must exclude low-relevance sections');
+  assert.equal(selection.text.includes('topped three regional charts'), true);
+  assert.equal(selection.text.includes('Boil the noodles'), false);
+  const chartPos = selection.text.indexOf('Chart performance');
+  const musicPos = selection.text.indexOf('Music releases');
+  if (musicPos !== -1 && chartPos !== -1) {
+    assert.ok(musicPos < chartPos, 'selected passages must follow document order');
+  }
+});
+
+test('scorePassages gives zero to empty queries without crashing', () => {
+  const passages = splitIntoPassages('## A\n\nhello world content here');
+  const scored = scorePassages(passages, '');
+  assert.equal(scored.length, 1);
+  assert.equal(scored[0].score, 0);
+});
+
+test('tokenizeForMatch segments Thai words and uses CJK bigrams', () => {
+  const thaiTokens = tokenizeForMatch('จวีจิ้งอีปล่อยเพลงใหม่', 'th');
+  assert.ok(thaiTokens.length > 1, 'Thai text must be segmented into multiple words');
+  const zhTokens = tokenizeForMatch('鞠婧祎新专辑', 'zh');
+  assert.ok(zhTokens.length >= 3, 'Chinese text must produce character bigrams');
+  const latin = tokenizeForMatch('Ju Jingyi chart', 'en');
+  assert.deepEqual(latin, ['ju', 'jingyi', 'chart']);
+});
+
+test('truncateByWords cuts at word boundaries instead of mid-word', () => {
+  const thaiText = 'จวีจิ้งอีปล่อยเพลงใหม่ล่าสุดพร้อมมิวสิกวิดีโอ '.repeat(20);
+  const truncated = truncateByWords(thaiText, 60, 'th');
+  assert.ok(truncated.length <= 60);
+  assert.ok(thaiText.startsWith(truncated));
+
+  const english = 'word '.repeat(100);
+  const truncatedEnglish = truncateByWords(english, 30, 'en');
+  assert.ok(truncatedEnglish.length <= 30);
+  assert.equal(truncatedEnglish.trimEnd().split(' ').pop(), 'word');
+});
+
+test('simhashText detects near-duplicate passages across pages', () => {
+  const original = 'Ju Jingyi released her newest single on September 1st alongside a full music video filmed in Chongqing.';
+  const paraphrase = 'Ju Jingyi released her newest single on September 1st alongside a full music video filmed in Chongqing!';
+  const different = 'The recipe for the soup requires boiling noodles for exactly four minutes before serving them hot.';
+
+  const state = { hashes: [] };
+  const first = filterUniquePassages([{ index: 0, heading: '', text: original }], state);
+  const duplicate = filterUniquePassages([{ index: 1, heading: '', text: paraphrase }], state);
+  const unique = filterUniquePassages([{ index: 2, heading: '', text: different }], state);
+
+  assert.equal(first.length, 1);
+  assert.equal(duplicate.length, 0, 'near-identical passage must be dropped');
+  assert.equal(unique.length, 1);
+
+  const deduped = dedupePassages([
+    { index: 0, heading: '', text: original },
+    { index: 1, heading: '', text: paraphrase },
+    { index: 2, heading: '', text: different },
+  ]);
+  assert.equal(deduped.length, 2);
+});
+
+test('hamming64 and simhashText behave on identical and disjoint input', () => {
+  const a = simhashText('identical passage text for hashing purposes here');
+  const b = simhashText('identical passage text for hashing purposes here');
+  assert.equal(hamming64(a, b), 0);
+  const c = simhashText('completely unrelated content about chess strategies and endgame play');
+  assert.ok(hamming64(a, c) > 6);
+});