@geminilight/mindos 0.6.72 → 0.6.73

package/_standalone/__tests__/core/tree-async.test.ts

@@ -0,0 +1,43 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { mkTempMindRoot, cleanupMindRoot, seedFile } from './helpers';
+import { collectAllFiles } from '@/lib/core/tree';
+import { collectAllFilesAsync } from '@/lib/core/tree';
+
+describe('collectAllFilesAsync', () => {
+  let mindRoot: string;
+
+  beforeEach(() => {
+    mindRoot = mkTempMindRoot();
+  });
+
+  afterEach(() => {
+    cleanupMindRoot(mindRoot);
+  });
+
+  it('returns same results as sync version', async () => {
+    seedFile(mindRoot, 'a.md', '# A');
+    seedFile(mindRoot, 'dir/b.md', '# B');
+    seedFile(mindRoot, 'dir/c.csv', 'col1,col2');
+    seedFile(mindRoot, 'ignored.txt', 'skip');
+
+    const syncResult = collectAllFiles(mindRoot).sort();
+    const asyncResult = (await collectAllFilesAsync(mindRoot)).sort();
+    expect(asyncResult).toEqual(syncResult);
+  });
+
+  it('handles empty directory', async () => {
+    const result = await collectAllFilesAsync(mindRoot);
+    expect(result).toEqual([]);
+  });
+
+  it('handles non-existent directory', async () => {
+    const result = await collectAllFilesAsync('/nonexistent/path');
+    expect(result).toEqual([]);
+  });
+
+  it('handles deep nesting', async () => {
+    seedFile(mindRoot, 'a/b/c/d.md', 'deep');
+    const result = await collectAllFilesAsync(mindRoot);
+    expect(result).toContain('a/b/c/d.md');
+  });
+});

package/_standalone/__tests__/core/tree.test.ts

@@ -0,0 +1,160 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { mkTempMindRoot, cleanupMindRoot, seedFile } from './helpers';
+import { getFileTree, collectAllFiles, renderTree, buildFileIndex } from '@/lib/core/tree';
+
+describe('tree', () => {
+  let mindRoot: string;
+
+  beforeEach(() => { mindRoot = mkTempMindRoot(); });
+  afterEach(() => { cleanupMindRoot(mindRoot); });
+
+  describe('getFileTree', () => {
+    it('returns empty for empty directory', () => {
+      expect(getFileTree(mindRoot)).toEqual([]);
+    });
+
+    it('includes .md and .csv files', () => {
+      seedFile(mindRoot, 'README.md', '# Hi');
+      seedFile(mindRoot, 'data.csv', 'a,b');
+      const tree = getFileTree(mindRoot);
+      expect(tree).toHaveLength(2);
+      expect(tree.map(n => n.name).sort()).toEqual(['README.md', 'data.csv']);
+    });
+
+    it('excludes non-allowed extensions', () => {
+      seedFile(mindRoot, 'test.md', 'md');
+      seedFile(mindRoot, 'test.txt', 'txt');
+      seedFile(mindRoot, 'test.js', 'js');
+      const tree = getFileTree(mindRoot);
+      expect(tree).toHaveLength(1);
+      expect(tree[0].name).toBe('test.md');
+    });
+
+    it('ignores .git and node_modules directories', () => {
+      seedFile(mindRoot, '.git/config', 'git');
+      seedFile(mindRoot, 'node_modules/pkg/index.md', 'npm');
+      seedFile(mindRoot, 'real.md', 'content');
+      const tree = getFileTree(mindRoot);
+      expect(tree).toHaveLength(1);
+      expect(tree[0].name).toBe('real.md');
+    });
+
+    it('builds nested directory structure', () => {
+      seedFile(mindRoot, 'Profile/Identity.md', 'me');
+      seedFile(mindRoot, 'Profile/Goals.md', 'goals');
+      const tree = getFileTree(mindRoot);
+      expect(tree).toHaveLength(1);
+      expect(tree[0].type).toBe('directory');
+      expect(tree[0].name).toBe('Profile');
+      expect(tree[0].children).toHaveLength(2);
+    });
+
+    it('omits empty directories', () => {
+      seedFile(mindRoot, 'empty/.gitkeep', '');
+      seedFile(mindRoot, 'has-md/file.md', 'content');
+      const tree = getFileTree(mindRoot);
+      expect(tree).toHaveLength(1);
+      expect(tree[0].name).toBe('has-md');
+    });
+
+    it('sorts directories before files, then alphabetically', () => {
+      seedFile(mindRoot, 'z.md', '');
+      seedFile(mindRoot, 'a.md', '');
+      seedFile(mindRoot, 'dir/x.md', '');
+      const tree = getFileTree(mindRoot);
+      expect(tree.map(n => n.name)).toEqual(['dir', 'a.md', 'z.md']);
+    });
+  });
+
+  describe('collectAllFiles', () => {
+    it('collects all relative paths', () => {
+      seedFile(mindRoot, 'a.md', '');
+      seedFile(mindRoot, 'sub/b.csv', '');
+      seedFile(mindRoot, 'sub/c.txt', '');
+      const files = collectAllFiles(mindRoot);
+      expect(files.sort()).toEqual(['a.md', 'sub/b.csv']);
+    });
+  });
+
+  describe('renderTree', () => {
+    it('renders a simple tree', () => {
+      seedFile(mindRoot, 'README.md', '');
+      seedFile(mindRoot, 'Profile/Identity.md', '');
+      const tree = getFileTree(mindRoot);
+      const rendered = renderTree(tree);
+      expect(rendered).toContain('Profile/');
+      expect(rendered).toContain('Identity.md');
+      expect(rendered).toContain('README.md');
+    });
+  });
+
+  describe('buildFileIndex', () => {
+    it('returns empty message for empty KB', () => {
+      expect(buildFileIndex(mindRoot)).toBe('(empty knowledge base)');
+    });
+
+    it('lists files with directory file counts', () => {
+      seedFile(mindRoot, 'Projects/roadmap.md', '');
+      seedFile(mindRoot, 'Projects/pricing.md', '');
+      seedFile(mindRoot, 'notes.md', '');
+      const index = buildFileIndex(mindRoot);
+      expect(index).toContain('Projects/ (2 files)');
+      expect(index).toContain('  roadmap.md');
+      expect(index).toContain('  pricing.md');
+      expect(index).toContain('notes.md');
+    });
+
+    it('respects maxDepth — collapses deep directories', () => {
+      seedFile(mindRoot, 'A/B/C/deep.md', '');
+      seedFile(mindRoot, 'A/B/shallow.md', '');
+
+      const index = buildFileIndex(mindRoot, { maxDepth: 1 });
+      expect(index).toContain('A/ (2 files)');
+      // depth 1 = we expand A/ but B/ should be collapsed
+      expect(index).toContain('  B/ (2 files)');
+      expect(index).not.toContain('C/');
+      expect(index).not.toContain('deep.md');
+    });
+
+    it('expands to depth 2 by default', () => {
+      seedFile(mindRoot, 'A/B/C/deep.md', '');
+      seedFile(mindRoot, 'A/B/mid.md', '');
+
+      const index = buildFileIndex(mindRoot);
+      // depth 0: A/, depth 1: B/, depth 2: C/ collapsed
+      expect(index).toContain('A/ (2 files)');
+      expect(index).toContain('  B/ (2 files)');
+      expect(index).toContain('    C/ (1 file)');
+      expect(index).toContain('    mid.md');
+      expect(index).not.toContain('deep.md');
+    });
+
+    it('truncates when directory has more files than maxFilesPerDir', () => {
+      for (let i = 0; i < 20; i++) {
+        seedFile(mindRoot, `file-${String(i).padStart(2, '0')}.md`, '');
+      }
+      const index = buildFileIndex(mindRoot, { maxFilesPerDir: 5 });
+      const lines = index.split('\n');
+      const fileLines = lines.filter(l => l.endsWith('.md'));
+      expect(fileLines).toHaveLength(5);
+      expect(index).toContain('... (15 more)');
+    });
+
+    it('counts nested files in directory totals', () => {
+      seedFile(mindRoot, 'Research/papers/a.md', '');
+      seedFile(mindRoot, 'Research/papers/b.md', '');
+      seedFile(mindRoot, 'Research/notes.md', '');
+      const index = buildFileIndex(mindRoot);
+      expect(index).toContain('Research/ (3 files)');
+    });
+
+    it('handles deeply nested directories at depth limit', () => {
+      seedFile(mindRoot, 'L1/L2/L3/L4/L5/deep.md', '');
+      const index = buildFileIndex(mindRoot, { maxDepth: 2 });
+      expect(index).toContain('L1/ (1 file)');
+      expect(index).toContain('  L2/ (1 file)');
+      expect(index).toContain('    L3/ (1 file)');
+      expect(index).not.toContain('L4');
+    });
+  });
+});

package/_standalone/__tests__/core/web-clip.test.ts

@@ -0,0 +1,229 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+import { clipUrl, isValidUrl } from '@/lib/core/web-clip';
+
+describe('isValidUrl', () => {
+  it('accepts http URLs', () => {
+    expect(isValidUrl('http://example.com')).toBe(true);
+    expect(isValidUrl('http://example.com/path?q=1')).toBe(true);
+  });
+
+  it('accepts https URLs', () => {
+    expect(isValidUrl('https://example.com')).toBe(true);
+    expect(isValidUrl('https://sub.example.com/path#section')).toBe(true);
+  });
+
+  it('rejects non-http schemes', () => {
+    expect(isValidUrl('ftp://example.com')).toBe(false);
+    expect(isValidUrl('file:///etc/passwd')).toBe(false);
+    expect(isValidUrl('javascript:alert(1)')).toBe(false);
+    expect(isValidUrl('data:text/html,<h1>hi</h1>')).toBe(false);
+  });
+
+  it('rejects empty and garbage strings', () => {
+    expect(isValidUrl('')).toBe(false);
+    expect(isValidUrl('not a url')).toBe(false);
+    expect(isValidUrl('example.com')).toBe(false);
+    expect(isValidUrl('   ')).toBe(false);
+  });
+
+  it('rejects null-like inputs passed as any', () => {
+    expect(isValidUrl(null as unknown as string)).toBe(false);
+    expect(isValidUrl(undefined as unknown as string)).toBe(false);
+  });
+});
+
+describe('clipUrl', () => {
+  beforeEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  it('rejects invalid URLs', async () => {
+    await expect(clipUrl('not-a-url')).rejects.toThrow('Invalid URL');
+    await expect(clipUrl('ftp://evil.com')).rejects.toThrow('Invalid URL');
+  });
+
+  it('clips a simple HTML page', async () => {
+    const html = `<!DOCTYPE html>
+<html><head><title>Test Article</title></head>
+<body>
+  <article>
+    <h1>Test Article</h1>
+    <p>This is a test article with enough content to pass the Readability threshold.
+       It needs to be longer than 100 characters to be parsed as an article by the
+       Readability library. So we add more text here to make sure it works properly
+       in our test environment.</p>
+    <p>Second paragraph with additional content to ensure the article extraction
+       algorithm has enough material to work with. We want at least a few hundred
+       characters of meaningful content.</p>
+  </article>
+</body></html>`;
+
+    const mockFetch = vi.fn().mockResolvedValue({
+      ok: true,
+      status: 200,
+      url: 'https://example.com/article',
+      headers: new Headers({
+        'content-type': 'text/html; charset=utf-8',
+      }),
+      text: () => Promise.resolve(html),
+    });
+    vi.stubGlobal('fetch', mockFetch);
+
+    const result = await clipUrl('https://example.com/article');
+
+    expect(result.title).toBe('Test Article');
+    expect(result.fileName).toBe('Test Article.md');
+    expect(result.url).toBe('https://example.com/article');
+    expect(result.markdown).toContain('# Test Article');
+    expect(result.markdown).toContain('---');
+    expect(result.markdown).toMatch(/source:.*example\.com\/article/);
+    expect(result.wordCount).toBeGreaterThan(0);
+  });
+
+  it('handles non-HTML content type', async () => {
+    const mockFetch = vi.fn().mockResolvedValue({
+      ok: true,
+      status: 200,
+      url: 'https://example.com/image.png',
+      headers: new Headers({
+        'content-type': 'image/png',
+      }),
+      text: () => Promise.resolve('binary data'),
+    });
+    vi.stubGlobal('fetch', mockFetch);
+
+    await expect(clipUrl('https://example.com/image.png'))
+      .rejects.toThrow('URL does not point to an HTML page');
+  });
+
+  it('handles HTTP error response', async () => {
+    const mockFetch = vi.fn().mockResolvedValue({
+      ok: false,
+      status: 404,
+      statusText: 'Not Found',
+      url: 'https://example.com/missing',
+      headers: new Headers({ 'content-type': 'text/html' }),
+    });
+    vi.stubGlobal('fetch', mockFetch);
+
+    await expect(clipUrl('https://example.com/missing'))
+      .rejects.toThrow('HTTP 404');
+  });
+
+  it('handles network errors', async () => {
+    const mockFetch = vi.fn().mockRejectedValue(new TypeError('fetch failed'));
+    vi.stubGlobal('fetch', mockFetch);
+
+    await expect(clipUrl('https://example.com'))
+      .rejects.toThrow('fetch failed');
+  });
+
+  it('handles pages with minimal content', async () => {
+    const html = `<!DOCTYPE html>
+<html><head><title>Minimal Page</title></head>
+<body><p>Short.</p></body></html>`;
+
+    const mockFetch = vi.fn().mockResolvedValue({
+      ok: true,
+      status: 200,
+      url: 'https://example.com/',
+      headers: new Headers({ 'content-type': 'text/html' }),
+      text: () => Promise.resolve(html),
+    });
+    vi.stubGlobal('fetch', mockFetch);
+
+    const result = await clipUrl('https://example.com/');
+
+    expect(result.title).toBe('Minimal Page');
+    expect(result.markdown).toContain('# Minimal Page');
+    expect(result.fileName).toBe('Minimal Page.md');
+  });
+
+  it('sanitizes titles with special characters', async () => {
+    const html = `<!DOCTYPE html>
+<html><head><title>My "Article" — with: special/chars</title></head>
+<body>
+  <article>
+    <h1>My "Article" — with: special/chars</h1>
+    <p>Long enough content to be picked up by Readability as a proper article.
+       We need several sentences of meaningful text content to ensure the extraction
+       works correctly. This is the third sentence adding more words.</p>
+  </article>
+</body></html>`;
+
+    const mockFetch = vi.fn().mockResolvedValue({
+      ok: true,
+      status: 200,
+      url: 'https://example.com/special',
+      headers: new Headers({ 'content-type': 'text/html' }),
+      text: () => Promise.resolve(html),
+    });
+    vi.stubGlobal('fetch', mockFetch);
+
+    const result = await clipUrl('https://example.com/special');
+
+    expect(result.fileName).not.toContain('/');
+    expect(result.fileName).not.toContain('"');
+    expect(result.fileName).not.toContain(':');
+    expect(result.fileName.endsWith('.md')).toBe(true);
+  });
+
+  it('includes frontmatter with source metadata', async () => {
+    const html = `<!DOCTYPE html>
+<html><head><title>Frontmatter Test</title></head>
+<body>
+  <article>
+    <h1>Frontmatter Test</h1>
+    <p>Content paragraph one with enough text to be extracted properly by
+       the Readability library. We need meaningful content here.</p>
+    <p>Content paragraph two adding more substance to the article.</p>
+  </article>
+</body></html>`;
+
+    const mockFetch = vi.fn().mockResolvedValue({
+      ok: true,
+      status: 200,
+      url: 'https://blog.example.com/post/123',
+      headers: new Headers({ 'content-type': 'text/html' }),
+      text: () => Promise.resolve(html),
+    });
+    vi.stubGlobal('fetch', mockFetch);
+
+    const result = await clipUrl('https://blog.example.com/post/123');
+
+    expect(result.markdown).toMatch(/^---\n/);
+    expect(result.markdown).toContain('title: Frontmatter Test');
+    expect(result.markdown).toMatch(/source:.*blog\.example\.com\/post\/123/);
+    expect(result.markdown).toContain('clipped:');
+    expect(result.siteName).toBe('blog.example.com');
+  });
+
+  it('handles CJK content word count', async () => {
+    const html = `<!DOCTYPE html>
+<html><head><title>中文文章</title></head>
+<body>
+  <article>
+    <h1>中文文章标题</h1>
+    <p>这是一篇中文文章的内容，包含足够多的中文字符来测试字数统计功能。
+       我们需要确保中日韩字符的计数逻辑正确工作，每个汉字算作一个词。
+       这段话应该有足够的内容让 Readability 提取器正常工作。</p>
+  </article>
+</body></html>`;
+
+    const mockFetch = vi.fn().mockResolvedValue({
+      ok: true,
+      status: 200,
+      url: 'https://example.cn/article',
+      headers: new Headers({ 'content-type': 'text/html; charset=utf-8' }),
+      text: () => Promise.resolve(html),
+    });
+    vi.stubGlobal('fetch', mockFetch);
+
+    const result = await clipUrl('https://example.cn/article');
+    expect(result.wordCount).toBeGreaterThan(20);
+  });
+});