bluera-knowledge 0.9.21

package/tests/integration/stress.test.ts

@@ -0,0 +1,326 @@
+/**
+ * Stress Tests
+ *
+ * Tests performance with moderate datasets (reduced scale for fast CI):
+ * - Multiple file indexing
+ * - Search performance
+ * - Sequential operation performance
+ * - Chunking behavior
+ *
+ * REWRITTEN: Now uses service APIs directly instead of CLI.
+ * Dataset sizes reduced to keep tests fast (30 files instead of 150, 100 lines instead of 1000).
+ */
+
+import { describe, it, expect, beforeAll, afterAll } from 'vitest';
+import { rm, mkdtemp, mkdir, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { Benchmark, measure } from '../helpers/performance-metrics';
+import { StoreService } from '../../src/services/store.service.js';
+import { IndexService } from '../../src/services/index.service.js';
+import { SearchService } from '../../src/services/search.service.js';
+import { LanceStore } from '../../src/db/lance.js';
+import { EmbeddingEngine } from '../../src/db/embeddings.js';
+import type { StoreId } from '../../src/types/brands.js';
+
+describe('Stress Tests', () => {
+  let tempDir: string;
+  let largeDatasetDir: string;
+  let storeService: StoreService;
+  let indexService: IndexService;
+  let searchService: SearchService;
+  let lanceStore: LanceStore;
+  let embeddingEngine: EmbeddingEngine;
+  let stressStoreId: StoreId;
+
+  // Reduced dataset sizes for fast tests
+  const FILE_COUNT = 30; // Reduced from 150
+  const LARGE_FILE_LINES = 100; // Reduced from 1000
+
+  beforeAll(async () => {
+    tempDir = await mkdtemp(join(tmpdir(), 'stress-test-'));
+    largeDatasetDir = join(tempDir, 'large-dataset');
+    await mkdir(largeDatasetDir, { recursive: true });
+
+    // Initialize services
+    lanceStore = new LanceStore(tempDir);
+    embeddingEngine = new EmbeddingEngine();
+    await embeddingEngine.initialize();
+
+    storeService = new StoreService(tempDir);
+    await storeService.initialize();
+
+    indexService = new IndexService(lanceStore, embeddingEngine);
+    searchService = new SearchService(lanceStore, embeddingEngine);
+  }, 30000);
+
+  afterAll(async () => {
+    await rm(tempDir, { recursive: true, force: true });
+  });
+
+  describe('Dataset Indexing', () => {
+    beforeAll(async () => {
+      // Generate test files with varied content (reduced to 30 files)
+      const categories = ['authentication', 'database', 'api', 'frontend', 'testing'];
+
+      const contentTemplates = [
+        (cat: string, i: number) =>
+          `# ${cat} Module ${i}\n\nThis module handles ${cat} functionality.\n\n## Overview\n\nThe ${cat} system provides essential features for the application.\n\n## Implementation\n\nKey features include:\n- Feature A for ${cat}\n- Feature B for ${cat}\n- Integration with other modules\n`,
+        (cat: string, i: number) =>
+          `export class ${cat}Service {\n  async process(data: ${cat}Data): Promise<${cat}Result> {\n    return { success: true };\n  }\n}\n`,
+        (cat: string, i: number) =>
+          `# ${cat} API\n\n## GET /${cat}/${i}\n\nRetrieve ${cat} data.\n\n## POST /${cat}/${i}\n\nCreate new ${cat} entry.\n`,
+      ];
+
+      let fileIndex = 0;
+      for (const category of categories) {
+        const categoryDir = join(largeDatasetDir, category);
+        await mkdir(categoryDir, { recursive: true });
+
+        for (let i = 0; i < Math.ceil(FILE_COUNT / categories.length); i++) {
+          const template = contentTemplates[fileIndex % contentTemplates.length];
+          const content = template(category, i);
+          const ext = fileIndex % 2 === 0 ? 'md' : 'ts';
+          await writeFile(join(categoryDir, `${category}-${i}.${ext}`), content);
+          fileIndex++;
+
+          if (fileIndex >= FILE_COUNT) break;
+        }
+        if (fileIndex >= FILE_COUNT) break;
+      }
+
+      // Create and index store
+      const storeResult = await storeService.create({
+        name: 'stress-store',
+        type: 'file',
+        path: largeDatasetDir,
+      });
+      if (!storeResult.success) throw storeResult.error;
+
+      stressStoreId = storeResult.data.id;
+      await lanceStore.initialize(stressStoreId);
+
+      const indexResult = await indexService.indexStore(storeResult.data);
+      if (!indexResult.success) throw indexResult.error;
+    }, 60000);
+
+    it(`indexes ${FILE_COUNT} files without timeout`, () => {
+      // Already indexed in beforeAll
+      expect(stressStoreId).toBeDefined();
+    });
+
+    it('verifies all documents were indexed', () => {
+      expect(stressStoreId).toBeDefined();
+    });
+  });
+
+  describe('Sequential Operation Performance', () => {
+    it('multiple sequential searches complete without timeout', async () => {
+      // Run 5 sequential searches - verify they complete, not relative performance
+      // (relative timing comparisons are too flaky across different CI hardware)
+      for (let i = 0; i < 5; i++) {
+        const result = await searchService.search({
+          query: 'module implementation',
+          stores: [stressStoreId],
+        });
+        expect(result.results).toBeDefined();
+      }
+    }, 30000);
+
+    it('alternating search modes complete without timeout', async () => {
+      const modes = ['vector', 'hybrid'] as const;
+
+      for (let i = 0; i < 6; i++) {
+        const mode = modes[i % modes.length];
+        const result = await searchService.search({
+          query: 'service implementation',
+          mode,
+          stores: [stressStoreId],
+        });
+        expect(result.results).toBeDefined();
+      }
+    }, 30000);
+  });
+
+  describe('Large File Chunking', () => {
+    let chunkingStoreId: StoreId;
+
+    beforeAll(async () => {
+      const largeFileDir = join(tempDir, 'large-files');
+      await mkdir(largeFileDir, { recursive: true });
+
+      // Create a file with 100 lines (reduced from 1000)
+      const lines: string[] = [];
+      for (let i = 0; i < LARGE_FILE_LINES; i++) {
+        lines.push(`## Section ${i}\n`);
+        lines.push(`This is content for section ${i}. It discusses various topics.\n`);
+        lines.push(`Key points:\n`);
+        lines.push(`- Point A about feature ${i}\n`);
+        lines.push(`- Point B about implementation ${i}\n\n`);
+      }
+
+      await writeFile(join(largeFileDir, 'large-document.md'), lines.join(''));
+
+      // Create and index store
+      const storeResult = await storeService.create({
+        name: 'chunking-store',
+        type: 'file',
+        path: largeFileDir,
+      });
+      if (!storeResult.success) throw storeResult.error;
+
+      chunkingStoreId = storeResult.data.id;
+      await lanceStore.initialize(chunkingStoreId);
+
+      const indexResult = await indexService.indexStore(storeResult.data);
+      if (!indexResult.success) throw indexResult.error;
+    }, 60000);
+
+    it('creates appropriate chunks for large files', () => {
+      expect(chunkingStoreId).toBeDefined();
+    });
+
+    it('searches across chunks efficiently', async () => {
+      const { measurement } = await measure('search-chunked', async () => {
+        return await searchService.search({
+          query: 'section implementation',
+          stores: [chunkingStoreId],
+        });
+      });
+
+      // Generous threshold - just catch catastrophic failures, not micro-benchmarks
+      expect(measurement.duration).toBeLessThan(10000);
+    });
+
+    it('finds content from different parts of large file', async () => {
+      const early = await searchService.search({
+        query: 'Section 10',
+        stores: [chunkingStoreId],
+      });
+      const middle = await searchService.search({
+        query: 'Section 50',
+        stores: [chunkingStoreId],
+      });
+      const late = await searchService.search({
+        query: 'Section 90',
+        stores: [chunkingStoreId],
+      });
+
+      const totalResults = early.results.length + middle.results.length + late.results.length;
+      expect(totalResults).toBeGreaterThan(0);
+    });
+  });
+
+  describe('Memory and Resource Usage', () => {
+    it('handles successive queries without degradation', async () => {
+      const rapidBenchmark = new Benchmark('rapid-queries');
+
+      // Run 10 queries (reduced from 30)
+      for (let i = 0; i < 10; i++) {
+        await rapidBenchmark.run(async () => {
+          return await searchService.search({
+            query: 'module',
+            limit: 5,
+            stores: [stressStoreId],
+          });
+        });
+      }
+
+      const stats = rapidBenchmark.getStats();
+      // Generous threshold - just catch catastrophic failures
+      expect(stats.max).toBeLessThan(30000);
+    }, 60000);
+  });
+
+  describe('Concurrent Store Operations', () => {
+    it('handles operations on multiple stores', async () => {
+      const store2Dir = join(tempDir, 'store2');
+      const store3Dir = join(tempDir, 'store3');
+      await mkdir(store2Dir, { recursive: true });
+      await mkdir(store3Dir, { recursive: true });
+
+      await writeFile(
+        join(store2Dir, 'doc.md'),
+        '# Store 2 Document\n\nContent for testing store 2 operations.'
+      );
+      await writeFile(
+        join(store3Dir, 'doc.md'),
+        '# Store 3 Document\n\nContent for testing store 3 operations.'
+      );
+
+      // Create stores
+      const store2Result = await storeService.create({
+        name: 'stress-store-2',
+        type: 'file',
+        path: store2Dir,
+      });
+      const store3Result = await storeService.create({
+        name: 'stress-store-3',
+        type: 'file',
+        path: store3Dir,
+      });
+
+      if (!store2Result.success || !store3Result.success) {
+        throw new Error('Failed to create stores');
+      }
+
+      const store2Id = store2Result.data.id;
+      const store3Id = store3Result.data.id;
+
+      await lanceStore.initialize(store2Id);
+      await lanceStore.initialize(store3Id);
+
+      await indexService.indexStore(store2Result.data);
+      await indexService.indexStore(store3Result.data);
+
+      // Search across stores
+      const { measurement } = await measure('multi-store-search', async () => {
+        return await searchService.search({
+          query: 'document content',
+          stores: [stressStoreId, store2Id, store3Id],
+        });
+      });
+
+      // Generous threshold - just catch catastrophic failures
+      expect(measurement.duration).toBeLessThan(30000);
+    }, 60000);
+  });
+
+  describe('Edge Case Stress', () => {
+    it('handles many small results efficiently', async () => {
+      const response = await searchService.search({
+        query: 'module',
+        limit: 50,
+        stores: [stressStoreId],
+      });
+
+      expect(response.results.length).toBeGreaterThan(0);
+      expect(response.results.length).toBeLessThanOrEqual(50);
+    });
+
+    it('handles broad queries without timeout', async () => {
+      const { measurement } = await measure('broad-query', async () => {
+        return await searchService.search({
+          query: 'the',
+          limit: 20,
+          stores: [stressStoreId],
+        });
+      });
+
+      // Generous threshold - just catch catastrophic failures
+      expect(measurement.duration).toBeLessThan(30000);
+    });
+
+    it('handles queries with many potential matches', async () => {
+      const { measurement } = await measure('common-term', async () => {
+        return await searchService.search({
+          query: 'implementation',
+          stores: [stressStoreId],
+        });
+      });
+
+      // Generous threshold - just catch catastrophic failures
+      expect(measurement.duration).toBeLessThan(10000);
+    });
+  });
+});

package/tests/mcp/server.test.ts

@@ -0,0 +1,15 @@
+import { describe, it, expect, beforeEach } from 'vitest';
+import { createMCPServer } from '../../src/mcp/server.js';
+
+describe('MCP Server', () => {
+  it('should create server with search tool', () => {
+    const server = createMCPServer({
+      dataDir: '/tmp/test',
+      config: undefined
+    });
+
+    expect(server).toBeDefined();
+    // Note: MCP SDK doesn't expose tools list directly for unit testing.
+    // Integration testing is performed manually via MCP inspector or client.
+  });
+});

package/tests/scripts/schemas/evaluation.json

@@ -0,0 +1,44 @@
+{
+  "type": "object",
+  "properties": {
+    "scores": {
+      "type": "object",
+      "properties": {
+        "relevance": { "type": "number", "minimum": 0, "maximum": 1 },
+        "ranking": { "type": "number", "minimum": 0, "maximum": 1 },
+        "coverage": { "type": "number", "minimum": 0, "maximum": 1 },
+        "snippetQuality": { "type": "number", "minimum": 0, "maximum": 1 },
+        "overall": { "type": "number", "minimum": 0, "maximum": 1 }
+      },
+      "required": ["relevance", "ranking", "coverage", "snippetQuality", "overall"]
+    },
+    "analysis": {
+      "type": "object",
+      "properties": {
+        "relevance": { "type": "string" },
+        "ranking": { "type": "string" },
+        "coverage": { "type": "string" },
+        "snippetQuality": { "type": "string" }
+      },
+      "required": ["relevance", "ranking", "coverage", "snippetQuality"]
+    },
+    "suggestions": {
+      "type": "array",
+      "items": { "type": "string" }
+    },
+    "resultAssessments": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "rank": { "type": "integer" },
+          "source": { "type": "string" },
+          "relevant": { "type": "boolean" },
+          "note": { "type": "string" }
+        },
+        "required": ["rank", "source", "relevant"]
+      }
+    }
+  },
+  "required": ["scores", "analysis", "suggestions", "resultAssessments"]
+}

package/tests/scripts/schemas/query-generation.json

@@ -0,0 +1,21 @@
+{
+  "type": "object",
+  "properties": {
+    "queries": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "query": { "type": "string" },
+          "intent": { "type": "string" },
+          "expectedContentTypes": {
+            "type": "array",
+            "items": { "type": "string" }
+          }
+        },
+        "required": ["query", "intent"]
+      }
+    }
+  },
+  "required": ["queries"]
+}

package/tests/services/code-unit.service.test.ts

@@ -0,0 +1,47 @@
+import { describe, it, expect } from 'vitest';
+import { CodeUnitService } from '../../src/services/code-unit.service.js';
+
+describe('CodeUnitService', () => {
+  it('should extract full function from TypeScript code', () => {
+    const code = `
+export function validateToken(token: string): boolean {
+  if (!token) return false;
+  return token.length > 0;
+}
+
+export function parseToken(token: string): object {
+  return JSON.parse(token);
+}
+`;
+
+    const service = new CodeUnitService();
+    const unit = service.extractCodeUnit(code, 'validateToken', 'typescript');
+
+    expect(unit).toBeDefined();
+    expect(unit.type).toBe('function');
+    expect(unit.name).toBe('validateToken');
+    expect(unit.signature).toBe('validateToken(token: string): boolean');
+    expect(unit.fullContent).toContain('export function validateToken');
+    expect(unit.startLine).toBe(2);
+    expect(unit.endLine).toBe(5);
+  });
+
+  it('should extract class with methods', () => {
+    const code = `
+export class UserService {
+  constructor(private repo: UserRepo) {}
+
+  async create(data: CreateUserData): Promise<User> {
+    return this.repo.save(data);
+  }
+}
+`;
+
+    const service = new CodeUnitService();
+    const unit = service.extractCodeUnit(code, 'UserService', 'typescript');
+
+    expect(unit.type).toBe('class');
+    expect(unit.name).toBe('UserService');
+    expect(unit.fullContent).toContain('class UserService');
+  });
+});

package/tests/services/search.progressive-context.test.ts

@@ -0,0 +1,35 @@
+import { describe, it, expect } from 'vitest';
+import type { SearchResult } from '../../src/types/search.js';
+
+describe('Progressive Context', () => {
+  it('should return minimal summary by default', () => {
+    const result: SearchResult = {
+      id: 'test-1',
+      score: 0.95,
+      content: 'function code here...',
+      metadata: {
+        type: 'chunk',
+        storeId: 'test-store' as any,
+        path: 'src/auth.ts',
+        indexedAt: new Date(),
+      },
+      summary: {
+        type: 'function',
+        name: 'validateToken',
+        signature: 'validateToken(token: string): boolean',
+        purpose: 'Validates JWT token',
+        location: 'src/auth.ts:45',
+        relevanceReason: 'Matches query about token validation'
+      }
+    };
+
+    expect(result.summary).toBeDefined();
+    expect(result.context).toBeUndefined();
+    expect(result.full).toBeUndefined();
+  });
+
+  it('should include context when detail=contextual', () => {
+    // Test that context layer is populated
+    expect(true).toBe(true); // Placeholder until implemented
+  });
+});

package/tsconfig.json

@@ -0,0 +1,34 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "lib": ["ES2022", "DOM"],
+    "outDir": "dist",
+    "declaration": true,
+    "declarationMap": true,
+    "sourceMap": true,
+    "strict": true,
+    "noImplicitAny": true,
+    "strictNullChecks": true,
+    "strictFunctionTypes": true,
+    "strictBindCallApply": true,
+    "strictPropertyInitialization": true,
+    "noImplicitThis": true,
+    "useUnknownInCatchVariables": true,
+    "alwaysStrict": true,
+    "noUnusedLocals": true,
+    "noUnusedParameters": true,
+    "noImplicitReturns": true,
+    "noFallthroughCasesInSwitch": true,
+    "noUncheckedIndexedAccess": true,
+    "noImplicitOverride": true,
+    "noPropertyAccessFromIndexSignature": true,
+    "exactOptionalPropertyTypes": true,
+    "esModuleInterop": true,
+    "forceConsistentCasingInFileNames": true,
+    "skipLibCheck": true
+  },
+  "include": ["src/**/*", "turndown-plugin-gfm.d.ts"],
+  "exclude": ["node_modules", "dist", "**/*.test.ts"]
+}

package/tsup.config.ts

@@ -0,0 +1,15 @@
+import { defineConfig } from 'tsup';
+
+export default defineConfig({
+  entry: {
+    index: 'src/index.ts',
+    'mcp/server': 'src/mcp/server.ts',
+    'workers/background-worker-cli': 'src/workers/background-worker-cli.ts',
+  },
+  format: ['esm'],
+  dts: true,
+  sourcemap: true,
+  clean: true,
+  target: 'node20',
+  shims: true,
+});

package/turndown-plugin-gfm.d.ts

@@ -0,0 +1,29 @@
+declare module 'turndown-plugin-gfm' {
+  import type TurndownService from 'turndown';
+
+  /**
+   * GitHub Flavored Markdown plugin for TurndownService
+   * Adds support for tables, strikethrough, task lists, and highlighted code blocks
+   */
+  export const gfm: TurndownService.Plugin;
+
+  /**
+   * Plugin for highlighted code blocks (e.g., <div class="highlight-source-js">)
+   */
+  export const highlightedCodeBlock: TurndownService.Plugin;
+
+  /**
+   * Plugin for strikethrough text (e.g., <del>, <s>, <strike>)
+   */
+  export const strikethrough: TurndownService.Plugin;
+
+  /**
+   * Plugin for GitHub-style tables
+   */
+  export const tables: TurndownService.Plugin;
+
+  /**
+   * Plugin for task list items (e.g., checkboxes in lists)
+   */
+  export const taskListItems: TurndownService.Plugin;
+}

package/vitest.config.ts

@@ -0,0 +1,79 @@
+import { defineConfig } from 'vitest/config';
+
+const coverageThreshold = 81;
+
+export default defineConfig({
+  test: {
+    // Separate unit and integration tests with different isolation settings
+    // Unit tests: fast parallel execution without isolation (use mocks only)
+    // Integration tests: safe execution with isolation (spawn processes, use filesystem)
+    projects: [
+      {
+        test: {
+          name: { label: 'unit', color: 'cyan' },
+          globals: true,
+          environment: 'node',
+          include: ['src/**/*.test.ts'],
+          // Exclude tests that spawn processes or have mock state conflicts
+          exclude: [
+            'src/crawl/bridge.test.ts',
+            'src/plugin/git-clone.test.ts',
+            'src/services/project-root.service.test.ts',
+          ],
+          // Use forks pool for onnxruntime-node compatibility
+          pool: 'forks',
+          maxWorkers: '75%',
+          // Fast: no isolation needed for unit tests (all use mocks)
+          isolate: false,
+        },
+      },
+      {
+        test: {
+          name: { label: 'integration', color: 'magenta' },
+          globals: true,
+          environment: 'node',
+          // Include all integration tests + tests that need isolation
+          include: [
+            'tests/**/*.test.ts',
+            'src/crawl/bridge.test.ts',
+            'src/plugin/git-clone.test.ts',
+            'src/services/project-root.service.test.ts',
+          ],
+          // Use forks pool for onnxruntime-node compatibility
+          pool: 'forks',
+          maxWorkers: '75%',
+          // Safe: isolation required (spawns processes, uses temp dirs)
+          isolate: true,
+        },
+      },
+    ],
+    coverage: {
+      provider: 'v8',
+      reporter: ['text', 'json', 'html'],
+      include: ['src/**/*.ts'],
+      exclude: [
+        'src/**/*.test.ts',
+        'src/index.ts',
+        // Barrel exports
+        'src/types/index.ts',
+        'src/services/index.ts',
+        'src/db/index.ts',
+        // Type-only files
+        'src/types/document.ts',
+        'src/types/job.ts',
+        'src/types/progress.ts',
+        'src/types/search.ts',
+        'src/mcp/types.ts',
+        // Entry points (CLI executables)
+        'src/server/index.ts',
+        'src/workers/background-worker-cli.ts',
+      ],
+      thresholds: {
+        lines: coverageThreshold,
+        branches: coverageThreshold,
+        functions: coverageThreshold,
+        statements: coverageThreshold,
+      },
+    },
+  },
+});