bluera-knowledge 0.9.21

package/tests/helpers/search-relevance.ts

@@ -0,0 +1,381 @@
+/**
+ * Search Relevance Utilities
+ *
+ * Helpers for validating search result quality and relevance.
+ */
+
+/**
+ * Search result item (matches CLI output structure)
+ */
+export interface SearchResult {
+  /** Result ranking position (1-based) */
+  rank: number;
+  /** Relevance score (0-1) */
+  score: number;
+  /** Source file path */
+  source: string;
+  /** Content snippet */
+  content: string;
+}
+
+/**
+ * Expected match for relevance testing
+ */
+export interface ExpectedMatch {
+  /** Keywords that should appear in content (any of these) */
+  keywords?: string[];
+  /** All of these keywords must appear */
+  requiredKeywords?: string[];
+  /** Source file should contain this string */
+  sourceContains?: string;
+  /** Minimum acceptable score */
+  minScore?: number;
+  /** Maximum acceptable rank position */
+  maxRank?: number;
+}
+
+/**
+ * Parse CLI search output into structured results
+ *
+ * @param output - Raw CLI output from search command
+ * @returns Array of parsed search results
+ *
+ * @example
+ * const output = cli('search "authentication"');
+ * const results = parseSearchOutput(output);
+ */
+export function parseSearchOutput(output: string): SearchResult[] {
+  const results: SearchResult[] = [];
+  const lines = output.split('\n');
+
+  let currentResult: Partial<SearchResult> | null = null;
+  let rank = 0;
+  let expectingLocation = false;
+  let locationSet = false;
+
+  for (const line of lines) {
+    // Match new format header: "1. [0.85] function: PROVIDERS"
+    // Or old format: "1. [0.85] path/to/file.ts"
+    const headerMatch = line.match(/^(\d+)\.\s+\[(-?[0-9.]+)\]\s+(.+)$/);
+    if (headerMatch) {
+      // Save previous result if it exists
+      if (currentResult) {
+        // If source is still empty, try to extract from content or use a placeholder
+        if (!currentResult.source) {
+          currentResult.source = 'unknown';
+        }
+        results.push(currentResult as SearchResult);
+      }
+
+      rank++;
+      const headerContent = headerMatch[3].trim();
+
+      // Check if this is new format (contains type prefix like "function:", "class:", etc.)
+      const typeMatch = headerContent.match(/^(function|class|interface|type|const|documentation):\s+(.+)$/);
+      if (typeMatch) {
+        // New format - next line should contain the location
+        currentResult = {
+          rank,
+          score: parseFloat(headerMatch[2]),
+          source: '', // Will be filled from next line
+          content: '',
+        };
+        expectingLocation = true;
+        locationSet = false;
+      } else {
+        // Old format - source is directly in header
+        currentResult = {
+          rank,
+          score: parseFloat(headerMatch[2]),
+          source: headerContent,
+          content: '',
+        };
+        expectingLocation = false;
+        locationSet = true;
+      }
+      continue;
+    }
+
+    // Process indented lines
+    if (currentResult && line.startsWith('   ')) {
+      const trimmed = line.trim();
+
+      // If we're expecting location and haven't set it yet
+      if (expectingLocation && !locationSet) {
+        // First indented line after header is the location
+        // Extract just the file path, removing line numbers if present
+        const pathMatch = trimmed.match(/^([^:]+?)(?::\d+)?$/);
+        if (pathMatch) {
+          currentResult.source = pathMatch[1];
+        } else {
+          currentResult.source = trimmed;
+        }
+        locationSet = true;
+        expectingLocation = false;
+        continue;
+      }
+
+      // Accumulate content lines (all other indented lines)
+      if (locationSet) {
+        currentResult.content += (currentResult.content ? '\n' : '') + trimmed;
+      }
+    }
+  }
+
+  // Don't forget the last result
+  if (currentResult) {
+    if (!currentResult.source) {
+      currentResult.source = 'unknown';
+    }
+    results.push(currentResult as SearchResult);
+  }
+
+  return results;
+}
+
+/**
+ * Check if a result matches expected criteria
+ *
+ * @param result - Search result to check
+ * @param expected - Expected match criteria
+ * @returns true if result matches all criteria
+ */
+export function matchesExpectation(
+  result: SearchResult,
+  expected: ExpectedMatch
+): boolean {
+  const contentLower = result.content.toLowerCase();
+  const sourceLower = result.source.toLowerCase();
+
+  // Check keywords (any match)
+  if (expected.keywords && expected.keywords.length > 0) {
+    const hasKeyword = expected.keywords.some((kw) =>
+      contentLower.includes(kw.toLowerCase())
+    );
+    if (!hasKeyword) return false;
+  }
+
+  // Check required keywords (all must match)
+  if (expected.requiredKeywords && expected.requiredKeywords.length > 0) {
+    const hasAllRequired = expected.requiredKeywords.every((kw) =>
+      contentLower.includes(kw.toLowerCase())
+    );
+    if (!hasAllRequired) return false;
+  }
+
+  // Check source path
+  if (expected.sourceContains) {
+    if (!sourceLower.includes(expected.sourceContains.toLowerCase())) {
+      return false;
+    }
+  }
+
+  // Check minimum score
+  if (expected.minScore !== undefined && result.score < expected.minScore) {
+    return false;
+  }
+
+  // Check maximum rank
+  if (expected.maxRank !== undefined && result.rank > expected.maxRank) {
+    return false;
+  }
+
+  return true;
+}
+
+/**
+ * Assert that results contain at least one match for expected criteria
+ *
+ * @param results - Search results to check
+ * @param expected - Expected match criteria
+ * @returns The first matching result, or throws if none found
+ */
+export function assertHasMatch(
+  results: SearchResult[],
+  expected: ExpectedMatch
+): SearchResult {
+  const match = results.find((r) => matchesExpectation(r, expected));
+  if (!match) {
+    const criteria = JSON.stringify(expected, null, 2);
+    const resultsSummary = results
+      .map((r) => `  ${r.rank}. [${r.score}] ${r.source}`)
+      .join('\n');
+    throw new Error(
+      `No result matches expected criteria:\n${criteria}\n\nResults:\n${resultsSummary}`
+    );
+  }
+  return match;
+}
+
+/**
+ * Assert that the top N results all match expected criteria
+ *
+ * @param results - Search results to check
+ * @param expected - Expected match criteria
+ * @param topN - Number of top results to check (default: 3)
+ */
+export function assertTopResultsMatch(
+  results: SearchResult[],
+  expected: ExpectedMatch,
+  topN = 3
+): void {
+  const topResults = results.slice(0, topN);
+
+  for (const result of topResults) {
+    if (!matchesExpectation(result, expected)) {
+      throw new Error(
+        `Top ${topN} result at rank ${result.rank} does not match expected criteria:\n` +
+          `Expected: ${JSON.stringify(expected)}\n` +
+          `Got: ${result.source} - "${result.content.slice(0, 100)}..."`
+      );
+    }
+  }
+}
+
+/**
+ * Calculate relevance metrics for a set of results
+ *
+ * @param results - Search results
+ * @param expected - Expected match criteria for "relevant" results
+ * @returns Relevance metrics
+ */
+export function calculateRelevanceMetrics(
+  results: SearchResult[],
+  expected: ExpectedMatch
+): {
+  precision: number;
+  relevantCount: number;
+  totalCount: number;
+  averageScore: number;
+  topScore: number;
+} {
+  const relevant = results.filter((r) => matchesExpectation(r, expected));
+
+  return {
+    precision: results.length > 0 ? relevant.length / results.length : 0,
+    relevantCount: relevant.length,
+    totalCount: results.length,
+    averageScore:
+      results.length > 0
+        ? results.reduce((sum, r) => sum + r.score, 0) / results.length
+        : 0,
+    topScore: results.length > 0 ? results[0].score : 0,
+  };
+}
+
+/**
+ * Compare relevance between two search result sets
+ *
+ * @param baseline - Baseline results
+ * @param comparison - Results to compare
+ * @param expected - Expected match criteria
+ * @returns Comparison metrics
+ */
+export function compareRelevance(
+  baseline: SearchResult[],
+  comparison: SearchResult[],
+  expected: ExpectedMatch
+): {
+  baselineMetrics: ReturnType<typeof calculateRelevanceMetrics>;
+  comparisonMetrics: ReturnType<typeof calculateRelevanceMetrics>;
+  precisionDiff: number;
+  scoreDiff: number;
+} {
+  const baselineMetrics = calculateRelevanceMetrics(baseline, expected);
+  const comparisonMetrics = calculateRelevanceMetrics(comparison, expected);
+
+  return {
+    baselineMetrics,
+    comparisonMetrics,
+    precisionDiff: comparisonMetrics.precision - baselineMetrics.precision,
+    scoreDiff: comparisonMetrics.topScore - baselineMetrics.topScore,
+  };
+}
+
+/**
+ * Assert minimum relevance score for results
+ *
+ * @param results - Search results
+ * @param minScore - Minimum acceptable score
+ */
+export function assertMinimumScores(
+  results: SearchResult[],
+  minScore: number
+): void {
+  for (const result of results) {
+    if (result.score < minScore) {
+      throw new Error(
+        `Result at rank ${result.rank} has score ${result.score}, ` +
+          `which is below minimum ${minScore}`
+      );
+    }
+  }
+}
+
+/**
+ * Check if results are properly ordered by score (descending)
+ */
+export function assertProperOrdering(results: SearchResult[]): void {
+  for (let i = 1; i < results.length; i++) {
+    if (results[i].score > results[i - 1].score) {
+      throw new Error(
+        `Results not properly ordered: rank ${i} has higher score ` +
+          `(${results[i].score}) than rank ${i - 1} (${results[i - 1].score})`
+      );
+    }
+  }
+}
+
+/**
+ * Common keyword sets for testing
+ */
+export const CommonKeywords = {
+  AUTHENTICATION: [
+    'auth',
+    'login',
+    'jwt',
+    'token',
+    'password',
+    'session',
+    'oauth',
+    'credential',
+  ],
+  API: [
+    'endpoint',
+    'request',
+    'response',
+    'rest',
+    'http',
+    'route',
+    'controller',
+    'middleware',
+  ],
+  DATABASE: [
+    'query',
+    'database',
+    'repository',
+    'entity',
+    'model',
+    'schema',
+    'sql',
+    'orm',
+  ],
+  TYPESCRIPT: [
+    'typescript',
+    'type',
+    'interface',
+    'generic',
+    'compiler',
+    'tsc',
+  ],
+  REACT: ['react', 'component', 'jsx', 'hook', 'state', 'props', 'render'],
+  ERROR_HANDLING: [
+    'error',
+    'exception',
+    'catch',
+    'throw',
+    'try',
+    'handler',
+    'validation',
+  ],
+} as const;

package/tests/integration/cli-consistency.test.ts

@@ -0,0 +1,299 @@
+import { describe, it, expect, beforeAll, afterAll } from 'vitest';
+import { execSync, spawnSync } from 'node:child_process';
+import { rm, mkdtemp, writeFile, mkdir } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+
+/**
+ * CLI Consistency Tests
+ *
+ * These tests verify:
+ * 1. Exit codes are consistent across commands
+ * 2. --format json is respected by all commands
+ * 3. --quiet suppresses non-essential output
+ * 4. Error messages follow consistent format
+ * 5. Spinners check TTY before displaying
+ * 6. store delete prompts for confirmation without --force
+ */
+
+/**
+ * CLI Consistency Tests
+ *
+ * Tests that CLI commands have consistent behavior:
+ * - Exit codes
+ * - JSON format support
+ * - Quiet mode
+ * - Error message format
+ *
+ * The CLI now properly cleans up resources via destroyServices() which
+ * stops the PythonBridge subprocess, allowing commands to exit cleanly.
+ */
+describe('CLI Consistency', () => {
+  let tempDir: string;
+  let testFilesDir: string;
+
+  beforeAll(async () => {
+    tempDir = await mkdtemp(join(tmpdir(), 'cli-consistency-'));
+    testFilesDir = join(tempDir, 'files');
+    await mkdir(testFilesDir, { recursive: true });
+    await writeFile(
+      join(testFilesDir, 'test.md'),
+      '# Test\n\nContent for testing CLI consistency.'
+    );
+  }, 30000);
+
+  afterAll(async () => {
+    await rm(tempDir, { recursive: true, force: true });
+  });
+
+  /**
+   * Helper to run CLI and capture output + exit code
+   * Properly handles quoted arguments
+   */
+  const runCli = (args: string, options: { expectError?: boolean; stdin?: string } = {}): {
+    stdout: string;
+    stderr: string;
+    exitCode: number;
+  } => {
+    // Parse args respecting quotes
+    const argArray: string[] = [];
+    let current = '';
+    let inQuotes = false;
+    let quoteChar = '';
+
+    for (const char of args) {
+      if ((char === '"' || char === "'") && !inQuotes) {
+        inQuotes = true;
+        quoteChar = char;
+      } else if (char === quoteChar && inQuotes) {
+        inQuotes = false;
+        quoteChar = '';
+      } else if (char === ' ' && !inQuotes) {
+        if (current) {
+          argArray.push(current);
+          current = '';
+        }
+      } else {
+        current += char;
+      }
+    }
+    if (current) {
+      argArray.push(current);
+    }
+
+    const result = spawnSync('node', ['dist/index.js', ...argArray, '--data-dir', tempDir], {
+      encoding: 'utf-8',
+      timeout: 60000,
+      input: options.stdin,
+    });
+    return {
+      stdout: result.stdout || '',
+      stderr: result.stderr || '',
+      exitCode: result.status ?? 1,
+    };
+  };
+
+  /**
+   * Helper for commands expected to succeed
+   */
+  const cli = (args: string): string => {
+    return execSync(`node dist/index.js ${args} --data-dir "${tempDir}"`, {
+      encoding: 'utf-8',
+      timeout: 60000,
+    });
+  };
+
+  describe('Exit Codes', () => {
+    it('returns exit code 0 on success', () => {
+      const result = runCli('store list');
+      expect(result.exitCode).toBe(0);
+    });
+
+    it('returns non-zero exit code when store not found', () => {
+      const result = runCli('store info nonexistent-store');
+      // Exit code may be 3 (expected) or 134 (SIGABRT from LanceDB cleanup)
+      expect(result.exitCode).not.toBe(0);
+      expect(result.stderr).toContain('Error: Store not found');
+    });
+
+    it('returns non-zero exit code when store not found for delete', () => {
+      const result = runCli('store delete nonexistent-store --force');
+      expect(result.exitCode).not.toBe(0);
+      expect(result.stderr).toContain('Error: Store not found');
+    });
+
+    it('returns non-zero exit code when store not found for index', () => {
+      const result = runCli('index nonexistent-store');
+      expect(result.exitCode).not.toBe(0);
+      expect(result.stderr).toContain('Error: Store not found');
+    });
+
+    it('returns non-zero exit code when store not found for search with --stores', () => {
+      const result = runCli('search "test" --stores nonexistent-store');
+      expect(result.exitCode).not.toBe(0);
+      expect(result.stderr).toContain('Error: Store not found');
+    });
+
+  });
+
+  describe('--format json Support', () => {
+    beforeAll(async () => {
+      // Create a store for testing
+      try {
+        cli(`store create json-test-store --type file --source "${testFilesDir}"`);
+        cli('index json-test-store');
+      } catch {
+        // Store may already exist
+      }
+    }, 120000);
+
+    it('store list supports --format json', () => {
+      const result = runCli('store list --format json');
+      expect(result.exitCode).toBe(0);
+      expect(() => JSON.parse(result.stdout)).not.toThrow();
+    });
+
+    it('store info supports --format json', () => {
+      const result = runCli('store info json-test-store --format json');
+      expect(result.exitCode).toBe(0);
+      const parsed = JSON.parse(result.stdout);
+      expect(parsed).toHaveProperty('name');
+      expect(parsed).toHaveProperty('id');
+    });
+
+    it('search supports --format json', () => {
+      const result = runCli('search "test" --format json');
+      expect(result.exitCode).toBe(0);
+      const parsed = JSON.parse(result.stdout);
+      expect(parsed).toHaveProperty('results');
+    });
+
+    it('index supports --format json', () => {
+      const result = runCli('index json-test-store --format json');
+      expect(result.exitCode).toBe(0);
+      const parsed = JSON.parse(result.stdout);
+      expect(parsed).toHaveProperty('documentsIndexed');
+    }, 120000);
+  });
+
+  describe('--quiet Flag', () => {
+    beforeAll(async () => {
+      try {
+        cli(`store create quiet-test-store --type file --source "${testFilesDir}"`);
+      } catch {
+        // Store may already exist
+      }
+    });
+
+    it('--quiet suppresses all output for index on success', () => {
+      const result = runCli('index quiet-test-store --quiet');
+      expect(result.exitCode).toBe(0);
+      // Quiet mode = no output on success
+      expect(result.stdout.trim()).toBe('');
+    }, 120000);
+
+    it('--quiet outputs only store names for store list', () => {
+      const result = runCli('store list --quiet');
+      expect(result.exitCode).toBe(0);
+      // Should not contain decorative headers
+      expect(result.stdout).not.toContain('Stores:');
+      // Should still list store names (one per line)
+      expect(result.stdout.trim().length).toBeGreaterThan(0);
+    });
+
+    it('--quiet outputs only paths for search', () => {
+      const result = runCli('search "test" --quiet');
+      expect(result.exitCode).toBe(0);
+      // Should not contain verbose headers
+      expect(result.stdout).not.toContain('Search:');
+      expect(result.stdout).not.toContain('Mode:');
+    });
+  });
+
+  describe('Error Message Format', () => {
+    it('uses consistent "Error:" prefix for store not found', () => {
+      const result = runCli('store info nonexistent');
+      expect(result.stderr).toMatch(/^Error: Store not found: nonexistent/m);
+    });
+
+    it('uses consistent "Error:" prefix for store delete not found', () => {
+      const result = runCli('store delete nonexistent --force');
+      expect(result.stderr).toMatch(/^Error: Store not found: nonexistent/m);
+    });
+
+    it('uses consistent "Error:" prefix for index not found', () => {
+      const result = runCli('index nonexistent');
+      expect(result.stderr).toMatch(/^Error: Store not found: nonexistent/m);
+    });
+
+    it('uses consistent "Error:" prefix for search store not found', () => {
+      const result = runCli('search "test" --stores nonexistent');
+      expect(result.stderr).toMatch(/^Error: Store not found: nonexistent/m);
+    });
+
+    it('uses consistent "Error:" prefix for crawl store not found', () => {
+      const result = runCli('crawl https://example.com nonexistent');
+      expect(result.stderr).toMatch(/^Error: /m);
+    });
+  });
+
+  describe('store delete Confirmation', () => {
+    beforeAll(async () => {
+      try {
+        cli(`store create delete-test-store --type file --source "${testFilesDir}"`);
+      } catch {
+        // Store may already exist
+      }
+    });
+
+    it('prompts for confirmation without --force', () => {
+      // When run without --force and without TTY, should fail or prompt
+      const result = runCli('store delete delete-test-store');
+      // Should either prompt (and fail due to no input) or require --force
+      expect(result.exitCode).not.toBe(0);
+      expect(result.stderr + result.stdout).toMatch(/confirm|--force|cancelled/i);
+    });
+
+    it('deletes without prompt with --force', () => {
+      // First recreate the store
+      try {
+        cli(`store create delete-force-store --type file --source "${testFilesDir}"`);
+      } catch {
+        // ignore
+      }
+
+      const result = runCli('store delete delete-force-store --force');
+      expect(result.exitCode).toBe(0);
+      expect(result.stdout).toContain('Deleted');
+    });
+
+    it('accepts -y as alias for --force', () => {
+      try {
+        cli(`store create delete-y-store --type file --source "${testFilesDir}"`);
+      } catch {
+        // ignore
+      }
+
+      const result = runCli('store delete delete-y-store -y');
+      expect(result.exitCode).toBe(0);
+      expect(result.stdout).toContain('Deleted');
+    });
+  });
+
+  describe('Non-TTY Spinner Behavior', () => {
+    it('does not show spinner characters in non-TTY mode for index', () => {
+      // Create a store for this test
+      try {
+        cli(`store create spinner-test --type file --source "${testFilesDir}"`);
+      } catch {
+        // ignore
+      }
+
+      const result = runCli('index spinner-test');
+      // Should not contain ora spinner frames (⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏ or similar)
+      expect(result.stdout).not.toMatch(/[⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏]/);
+      expect(result.exitCode).toBe(0);
+    }, 120000);
+
+  });
+});