bluera-knowledge 0.9.21

package/src/services/chunking.service.test.ts

@@ -0,0 +1,363 @@
+import { describe, it, expect } from 'vitest';
+import { ChunkingService } from './chunking.service.js';
+
+describe('ChunkingService', () => {
+  const chunker = new ChunkingService({ chunkSize: 100, chunkOverlap: 20 });
+
+  describe('Basic sliding window chunking', () => {
+    it('splits text into chunks', () => {
+      const text = 'A'.repeat(250);
+      const chunks = chunker.chunk(text);
+      expect(chunks.length).toBeGreaterThan(1);
+      expect(chunks.every(c => c.content.length <= 100)).toBe(true);
+    });
+
+    it('preserves overlap between chunks', () => {
+      const text = 'word '.repeat(50); // 250 chars
+      const chunks = chunker.chunk(text);
+      if (chunks.length >= 2) {
+        const end1 = chunks[0]!.content.slice(-20);
+        const start2 = chunks[1]!.content.slice(0, 20);
+        expect(end1).toBe(start2);
+      }
+    });
+
+    it('returns single chunk for small text', () => {
+      const text = 'small text';
+      const chunks = chunker.chunk(text);
+      expect(chunks).toHaveLength(1);
+      expect(chunks[0]!.content).toBe(text);
+    });
+
+    it('assigns chunk indices correctly', () => {
+      const text = 'A'.repeat(300);
+      const chunks = chunker.chunk(text);
+      expect(chunks[0]!.chunkIndex).toBe(0);
+      expect(chunks[1]!.chunkIndex).toBe(1);
+      expect(chunks.every(c => c.totalChunks === chunks.length)).toBe(true);
+    });
+
+    it('handles empty text', () => {
+      const chunks = chunker.chunk('');
+      expect(chunks).toHaveLength(1);
+      expect(chunks[0]!.content).toBe('');
+    });
+
+    it('sets correct offsets for chunks', () => {
+      const text = 'A'.repeat(250);
+      const chunks = chunker.chunk(text);
+      expect(chunks[0]!.startOffset).toBe(0);
+      expect(chunks[0]!.endOffset).toBe(100);
+      expect(chunks[1]!.startOffset).toBe(80); // chunkSize - overlap
+      expect(chunks[1]!.endOffset).toBe(180);
+    });
+  });
+
+  describe('Markdown semantic chunking', () => {
+    it('chunks markdown by sections', () => {
+      const markdown = `# Header 1
+Content for section 1
+
+## Header 2
+Content for section 2
+
+### Header 3
+Content for section 3`;
+
+      const chunks = chunker.chunk(markdown, 'test.md');
+      expect(chunks.length).toBeGreaterThan(0);
+      expect(chunks.some(c => c.sectionHeader === 'Header 1')).toBe(true);
+      expect(chunks.some(c => c.sectionHeader === 'Header 2')).toBe(true);
+      expect(chunks.some(c => c.sectionHeader === 'Header 3')).toBe(true);
+    });
+
+    it('handles markdown with no headers', () => {
+      const markdown = 'Just plain text without headers';
+      const chunks = chunker.chunk(markdown, 'test.md');
+      expect(chunks).toHaveLength(1);
+      expect(chunks[0]!.sectionHeader).toBeUndefined();
+    });
+
+    it('handles markdown with only one header', () => {
+      const markdown = '# Only Header\nSome content';
+      const chunks = chunker.chunk(markdown, 'test.md');
+      expect(chunks.length).toBeGreaterThan(0);
+    });
+
+    it('splits large markdown sections with sliding window', () => {
+      const largeContent = 'A'.repeat(150);
+      const markdown = `# Header 1\n${largeContent}\n\n## Header 2\nSmall content`;
+
+      const chunks = chunker.chunk(markdown, 'test.md');
+      // Large section should be split
+      expect(chunks.length).toBeGreaterThan(2);
+      expect(chunks.filter(c => c.sectionHeader === 'Header 1').length).toBeGreaterThan(1);
+    });
+
+    it('preserves markdown section headers in metadata', () => {
+      const markdown = `#### Level 4 Header
+Content here`;
+
+      const chunks = chunker.chunk(markdown, 'test.md');
+      expect(chunks[0]!.sectionHeader).toBe('Level 4 Header');
+    });
+
+    it('handles consecutive headers', () => {
+      const markdown = `# Header 1
+## Header 2
+### Header 3
+Content`;
+
+      const chunks = chunker.chunk(markdown, 'test.md');
+      expect(chunks.length).toBeGreaterThan(0);
+    });
+
+    it('handles markdown with mixed header levels', () => {
+      const markdown = `# H1
+Content 1
+### H3
+Content 3
+## H2
+Content 2`;
+
+      const chunks = chunker.chunk(markdown, 'test.md');
+      expect(chunks.some(c => c.sectionHeader === 'H1')).toBe(true);
+      expect(chunks.some(c => c.sectionHeader === 'H3')).toBe(true);
+      expect(chunks.some(c => c.sectionHeader === 'H2')).toBe(true);
+    });
+  });
+
+  describe('Code semantic chunking', () => {
+    it('chunks code by function declarations', () => {
+      const code = `function foo() {
+  return 'foo';
+}
+
+function bar() {
+  return 'bar';
+}`;
+
+      const chunks = chunker.chunk(code, 'test.ts');
+      expect(chunks.length).toBeGreaterThan(0);
+      expect(chunks.some(c => c.functionName === 'foo')).toBe(true);
+      expect(chunks.some(c => c.functionName === 'bar')).toBe(true);
+    });
+
+    it('handles class declarations', () => {
+      const code = `class MyClass {
+  constructor() {}
+}
+
+class AnotherClass {
+  method() {}
+}`;
+
+      const chunks = chunker.chunk(code, 'test.ts');
+      expect(chunks.some(c => c.functionName === 'MyClass')).toBe(true);
+      expect(chunks.some(c => c.functionName === 'AnotherClass')).toBe(true);
+    });
+
+    it('handles exported declarations', () => {
+      const code = `export function exportedFn() {
+  return 1;
+}
+
+export class ExportedClass {}`;
+
+      const chunks = chunker.chunk(code, 'test.ts');
+      expect(chunks.some(c => c.functionName === 'exportedFn')).toBe(true);
+      expect(chunks.some(c => c.functionName === 'ExportedClass')).toBe(true);
+    });
+
+    it('handles async functions', () => {
+      const code = `async function asyncFn() {
+  return await Promise.resolve(1);
+}`;
+
+      const chunks = chunker.chunk(code, 'test.ts');
+      expect(chunks.some(c => c.functionName === 'asyncFn')).toBe(true);
+    });
+
+    it('handles const/let/var declarations', () => {
+      const code = `const myConst = 42;
+
+let myLet = 'test';
+
+var myVar = true;`;
+
+      const chunks = chunker.chunk(code, 'test.ts');
+      expect(chunks.some(c => c.functionName === 'myConst')).toBe(true);
+      expect(chunks.some(c => c.functionName === 'myLet')).toBe(true);
+      expect(chunks.some(c => c.functionName === 'myVar')).toBe(true);
+    });
+
+    it('handles interface and type declarations', () => {
+      const code = `interface MyInterface {
+  prop: string;
+}
+
+type MyType = {
+  value: number;
+}`;
+
+      const chunks = chunker.chunk(code, 'test.ts');
+      expect(chunks.some(c => c.functionName === 'MyInterface')).toBe(true);
+      expect(chunks.some(c => c.functionName === 'MyType')).toBe(true);
+    });
+
+    it('handles enum declarations', () => {
+      const code = `enum Color {
+  Red,
+  Green,
+  Blue
+}`;
+
+      const chunks = chunker.chunk(code, 'test.ts');
+      expect(chunks.some(c => c.functionName === 'Color')).toBe(true);
+    });
+
+    it('handles code with no declarations', () => {
+      const code = 'console.log("hello");';
+      const chunks = chunker.chunk(code, 'test.ts');
+      expect(chunks).toHaveLength(1);
+    });
+
+    it('handles code with only one declaration', () => {
+      const code = 'function single() { return 1; }';
+      const chunks = chunker.chunk(code, 'test.ts');
+      expect(chunks).toHaveLength(1);
+    });
+
+    it('splits large function declarations with sliding window', () => {
+      const largeBody = 'console.log("test");\\n'.repeat(20);
+      const code = `function largeFn() {
+${largeBody}
+}
+
+function smallFn() {
+  return 1;
+}`;
+
+      const chunks = chunker.chunk(code, 'test.ts');
+      expect(chunks.length).toBeGreaterThan(2);
+    });
+
+    it('handles nested braces in code', () => {
+      const code = `function withNested() {
+  if (true) {
+    while (x) {
+      return { a: 1 };
+    }
+  }
+}`;
+
+      const chunks = chunker.chunk(code, 'test.ts');
+      expect(chunks.some(c => c.functionName === 'withNested')).toBe(true);
+    });
+
+    it('exposes brace counting bug - braces in strings', () => {
+      // This test should FAIL with the current implementation
+      // because it doesn't handle braces inside strings
+      const code = `function withString() {
+  const obj = "{}";
+  const template = \`{ value: \${x} }\`;
+  return true;
+}
+
+function nextFn() {
+  return 2;
+}`;
+
+      const chunks = chunker.chunk(code, 'test.ts');
+      expect(chunks.some(c => c.functionName === 'withString')).toBe(true);
+      expect(chunks.some(c => c.functionName === 'nextFn')).toBe(true);
+      // The bug: braces in strings should be ignored for boundary detection
+    });
+
+    it('exposes brace counting bug - braces in comments', () => {
+      // This test should FAIL with the current implementation
+      // because it doesn't handle braces in comments
+      const code = `function withComments() {
+  // This is a comment with { and }
+  /* Multi-line comment
+   * with { braces } inside
+   */
+  return true;
+}
+
+function nextFn() {
+  return 2;
+}`;
+
+      const chunks = chunker.chunk(code, 'test.ts');
+      expect(chunks.some(c => c.functionName === 'withComments')).toBe(true);
+      expect(chunks.some(c => c.functionName === 'nextFn')).toBe(true);
+      // The bug: braces in comments should be ignored for boundary detection
+    });
+
+    it('handles JSDoc comments before declarations', () => {
+      const code = `/**
+ * Documentation for myFn
+ * @returns A number
+ */
+function myFn() {
+  return 42;
+}`;
+
+      const chunks = chunker.chunk(code, 'test.ts');
+      expect(chunks.some(c => c.functionName === 'myFn')).toBe(true);
+      expect(chunks[0]!.content).toContain('Documentation');
+    });
+  });
+
+  describe('File type detection', () => {
+    it('uses markdown chunking for .md files', () => {
+      const text = '# Header\nContent';
+      const chunks = chunker.chunk(text, 'readme.md');
+      expect(chunks[0]!.sectionHeader).toBe('Header');
+    });
+
+    it('uses markdown chunking for .MD files (case insensitive)', () => {
+      const text = '# Header\nContent';
+      const chunks = chunker.chunk(text, 'README.MD');
+      expect(chunks[0]!.sectionHeader).toBe('Header');
+    });
+
+    it('uses code chunking for .ts files', () => {
+      const code = 'function test() {}\nfunction test2() {}';
+      const chunks = chunker.chunk(code, 'file.ts');
+      expect(chunks.some(c => c.functionName === 'test')).toBe(true);
+    });
+
+    it('uses code chunking for .tsx files', () => {
+      const code = 'function Component() {}\nfunction Other() {}';
+      const chunks = chunker.chunk(code, 'component.tsx');
+      expect(chunks.some(c => c.functionName === 'Component')).toBe(true);
+    });
+
+    it('uses code chunking for .js files', () => {
+      const code = 'function test() {}\nfunction test2() {}';
+      const chunks = chunker.chunk(code, 'file.js');
+      expect(chunks.some(c => c.functionName === 'test')).toBe(true);
+    });
+
+    it('uses code chunking for .jsx files', () => {
+      const code = 'function Component() {}\nfunction Other() {}';
+      const chunks = chunker.chunk(code, 'component.jsx');
+      expect(chunks.some(c => c.functionName === 'Component')).toBe(true);
+    });
+
+    it('uses sliding window for unknown file types', () => {
+      const text = '# Not Markdown\nfunction notCode() {}';
+      const chunks = chunker.chunk(text, 'file.txt');
+      expect(chunks[0]!.sectionHeader).toBeUndefined();
+      expect(chunks[0]!.functionName).toBeUndefined();
+    });
+
+    it('uses sliding window when no file path provided', () => {
+      const text = 'Some text';
+      const chunks = chunker.chunk(text);
+      expect(chunks).toHaveLength(1);
+    });
+  });
+});

package/src/services/chunking.service.ts

@@ -0,0 +1,350 @@
+export interface ChunkConfig {
+  chunkSize: number;
+  chunkOverlap: number;
+}
+
+export interface Chunk {
+  content: string;
+  chunkIndex: number;
+  totalChunks: number;
+  startOffset: number;
+  endOffset: number;
+  /** Section header if this chunk starts a markdown section */
+  sectionHeader?: string | undefined;
+  /** Function or class name if this chunk contains a code declaration */
+  functionName?: string | undefined;
+  /** JSDoc/comment summary extracted from this chunk */
+  docSummary?: string | undefined;
+}
+
+export class ChunkingService {
+  private readonly chunkSize: number;
+  private readonly chunkOverlap: number;
+
+  constructor(config: ChunkConfig) {
+    this.chunkSize = config.chunkSize;
+    this.chunkOverlap = config.chunkOverlap;
+  }
+
+  /**
+   * Chunk text content. Uses semantic chunking for Markdown and code files,
+   * falling back to sliding window for other content.
+   */
+  chunk(text: string, filePath?: string): Chunk[] {
+    // Use semantic chunking for Markdown files
+    if (filePath !== undefined && filePath !== '' && /\.md$/i.test(filePath)) {
+      return this.chunkMarkdown(text);
+    }
+
+    // Use semantic chunking for TypeScript/JavaScript files
+    if (filePath !== undefined && filePath !== '' && /\.(ts|tsx|js|jsx)$/i.test(filePath)) {
+      return this.chunkCode(text);
+    }
+
+    return this.chunkSlidingWindow(text);
+  }
+
+  /**
+   * Semantic chunking for Markdown files.
+   * Splits on section headers to keep related content together.
+   */
+  private chunkMarkdown(text: string): Chunk[] {
+    // Match markdown headers (# through ####)
+    const headerRegex = /^(#{1,4})\s+(.+)$/gm;
+    const sections: Array<{ header: string; content: string; startOffset: number }> = [];
+
+    let lastIndex = 0;
+    let lastHeader = '';
+    let match: RegExpExecArray | null;
+
+    while ((match = headerRegex.exec(text)) !== null) {
+      // Save previous section
+      if (match.index > lastIndex) {
+        const content = text.slice(lastIndex, match.index).trim();
+        if (content) {
+          sections.push({
+            header: lastHeader,
+            content: content,
+            startOffset: lastIndex,
+          });
+        }
+      }
+      lastHeader = match[2] ?? '';
+      lastIndex = match.index;
+    }
+
+    // Add final section
+    const finalContent = text.slice(lastIndex).trim();
+    if (finalContent) {
+      sections.push({
+        header: lastHeader,
+        content: finalContent,
+        startOffset: lastIndex,
+      });
+    }
+
+    // If no sections found, fall back to sliding window
+    if (sections.length === 0) {
+      return this.chunkSlidingWindow(text);
+    }
+
+    // Convert sections to chunks, splitting large sections if needed
+    const chunks: Chunk[] = [];
+
+    for (const section of sections) {
+      if (section.content.length <= this.chunkSize) {
+        // Section fits in one chunk
+        chunks.push({
+          content: section.content,
+          chunkIndex: chunks.length,
+          totalChunks: 0,
+          startOffset: section.startOffset,
+          endOffset: section.startOffset + section.content.length,
+          sectionHeader: section.header || undefined,
+        });
+      } else {
+        // Split large section using sliding window
+        const sectionChunks = this.chunkSlidingWindow(section.content);
+        for (const subChunk of sectionChunks) {
+          chunks.push({
+            ...subChunk,
+            chunkIndex: chunks.length,
+            startOffset: section.startOffset + subChunk.startOffset,
+            endOffset: section.startOffset + subChunk.endOffset,
+            sectionHeader: section.header || undefined,
+          });
+        }
+      }
+    }
+
+    // Set totalChunks
+    for (const chunk of chunks) {
+      chunk.totalChunks = chunks.length;
+    }
+
+    return chunks;
+  }
+
+  /**
+   * Semantic chunking for TypeScript/JavaScript code files.
+   * Splits on top-level declarations to keep functions/classes together.
+   */
+  private chunkCode(text: string): Chunk[] {
+    // Match top-level declarations with optional JSDoc/comments before them
+    const declarationRegex = /^(?:\/\*\*[\s\S]*?\*\/\s*)?(?:export\s+)?(?:async\s+)?(?:function|class|interface|type|const|let|var|enum)\s+(\w+)/gm;
+    const declarations: Array<{ startOffset: number; endOffset: number; name?: string }> = [];
+
+    let match: RegExpExecArray | null;
+    while ((match = declarationRegex.exec(text)) !== null) {
+      const name = match[1];
+      const decl: { startOffset: number; endOffset: number; name?: string } = {
+        startOffset: match.index,
+        endOffset: match.index,
+      };
+      if (name !== undefined) {
+        decl.name = name;
+      }
+      declarations.push(decl);
+    }
+
+    // If no declarations found, use sliding window
+    if (declarations.length === 0) {
+      return this.chunkSlidingWindow(text);
+    }
+
+    // Find end of each declaration using brace-aware boundary detection
+    for (let i = 0; i < declarations.length; i++) {
+      const currentDecl = declarations[i];
+      const nextDecl = declarations[i + 1];
+      if (currentDecl === undefined) continue;
+
+      // For declarations that likely have braces (functions, classes, enums)
+      // use smart boundary detection
+      const declText = text.slice(currentDecl.startOffset);
+      if (/^(?:\/\*\*[\s\S]*?\*\/\s*)?(?:export\s+)?(?:async\s+)?(?:function|class|enum)\s+/m.test(declText)) {
+        const boundary = this.findDeclarationEnd(declText);
+        if (boundary > 0) {
+          currentDecl.endOffset = currentDecl.startOffset + boundary;
+        } else {
+          // Fall back to next declaration or EOF
+          currentDecl.endOffset = nextDecl !== undefined ? nextDecl.startOffset : text.length;
+        }
+      } else {
+        // For other declarations (interface, type, const, let, var), use next declaration or EOF
+        currentDecl.endOffset = nextDecl !== undefined ? nextDecl.startOffset : text.length;
+      }
+    }
+
+    const chunks: Chunk[] = [];
+
+    for (const decl of declarations) {
+      const content = text.slice(decl.startOffset, decl.endOffset).trim();
+
+      if (content.length <= this.chunkSize) {
+        // Declaration fits in one chunk
+        chunks.push({
+          content,
+          chunkIndex: chunks.length,
+          totalChunks: 0,
+          startOffset: decl.startOffset,
+          endOffset: decl.endOffset,
+          functionName: decl.name,
+        });
+      } else {
+        // Split large declaration with sliding window
+        const declChunks = this.chunkSlidingWindow(content);
+        for (const subChunk of declChunks) {
+          chunks.push({
+            ...subChunk,
+            chunkIndex: chunks.length,
+            startOffset: decl.startOffset + subChunk.startOffset,
+            endOffset: decl.startOffset + subChunk.endOffset,
+            functionName: decl.name,
+          });
+        }
+      }
+    }
+
+    // Set totalChunks
+    for (const chunk of chunks) {
+      chunk.totalChunks = chunks.length;
+    }
+
+    return chunks.length > 0 ? chunks : this.chunkSlidingWindow(text);
+  }
+
+  /**
+   * Find the end of a code declaration by counting braces while ignoring
+   * braces inside strings and comments.
+   * Returns the offset where the declaration ends, or -1 if not found.
+   */
+  private findDeclarationEnd(text: string): number {
+    let braceCount = 0;
+    let inString = false;
+    let inSingleLineComment = false;
+    let inMultiLineComment = false;
+    let stringChar = '';
+    let i = 0;
+    let foundFirstBrace = false;
+
+    // Find the first opening brace
+    while (i < text.length) {
+      const char = text[i];
+      const nextChar = i + 1 < text.length ? text[i + 1] : '';
+
+      // Handle comments
+      if (!inString && !inMultiLineComment && char === '/' && nextChar === '/') {
+        inSingleLineComment = true;
+        i += 2;
+        continue;
+      }
+
+      if (!inString && !inSingleLineComment && char === '/' && nextChar === '*') {
+        inMultiLineComment = true;
+        i += 2;
+        continue;
+      }
+
+      if (inMultiLineComment && char === '*' && nextChar === '/') {
+        inMultiLineComment = false;
+        i += 2;
+        continue;
+      }
+
+      if (inSingleLineComment && char === '\n') {
+        inSingleLineComment = false;
+        i++;
+        continue;
+      }
+
+      // Skip if in comment
+      if (inSingleLineComment || inMultiLineComment) {
+        i++;
+        continue;
+      }
+
+      // Handle strings
+      if (!inString && (char === '"' || char === "'" || char === '`')) {
+        inString = true;
+        stringChar = char;
+        i++;
+        continue;
+      }
+
+      if (inString && char === '\\') {
+        // Skip escaped character
+        i += 2;
+        continue;
+      }
+
+      if (inString && char === stringChar) {
+        inString = false;
+        stringChar = '';
+        i++;
+        continue;
+      }
+
+      // Skip if in string
+      if (inString) {
+        i++;
+        continue;
+      }
+
+      // Count braces
+      if (char === '{') {
+        braceCount++;
+        foundFirstBrace = true;
+      } else if (char === '}') {
+        braceCount--;
+        if (foundFirstBrace && braceCount === 0) {
+          // Found the closing brace
+          return i + 1;
+        }
+      }
+
+      i++;
+    }
+
+    // If we didn't find a complete declaration, return -1
+    return -1;
+  }
+
+  /**
+   * Traditional sliding window chunking for non-Markdown content.
+   */
+  private chunkSlidingWindow(text: string): Chunk[] {
+    if (text.length <= this.chunkSize) {
+      return [{
+        content: text,
+        chunkIndex: 0,
+        totalChunks: 1,
+        startOffset: 0,
+        endOffset: text.length,
+      }];
+    }
+
+    const chunks: Chunk[] = [];
+    const step = this.chunkSize - this.chunkOverlap;
+    let start = 0;
+
+    while (start < text.length) {
+      const end = Math.min(start + this.chunkSize, text.length);
+      chunks.push({
+        content: text.slice(start, end),
+        chunkIndex: chunks.length,
+        totalChunks: 0,
+        startOffset: start,
+        endOffset: end,
+      });
+      start += step;
+      if (end === text.length) break;
+    }
+
+    // Set totalChunks
+    for (const chunk of chunks) {
+      chunk.totalChunks = chunks.length;
+    }
+
+    return chunks;
+  }
+}