gitforest 0.1.0

package/docs/ai/implementation-guide.md

@@ -0,0 +1,776 @@
+# Gitforest Implementation Guide - All Improvements
+
+## Phase 1: Critical Fixes (Week 1)
+
+### 1. TKT-001: Fix Database Error
+**File:** `src/db/index.ts`
+**Issue:** Missing `await` in clearCache function
+
+```typescript
+// Current (broken)
+export async function clearCache(): Promise<void> {
+  const database = await initDb();
+  database.delete(schema.projects).all();  // Missing await
+  database.delete(schema.remoteStatus).all();  // Missing await
+}
+
+// Fixed
+export async function clearCache(): Promise<void> {
+  try {
+    const database = await initDb();
+    
+    // Run in parallel for better performance
+    await Promise.all([
+      database.delete(schema.projects).all(),
+      database.delete(schema.remoteStatus).all()
+    ]);
+  } catch (error) {
+    console.error('Failed to clear cache:', error);
+    throw new Error(`Cache clear failed: ${error instanceof Error ? error.message : String(error)}`);
+  }
+}
+```
+
+### 2. TKT-002: Add Graceful Shutdown
+**File:** `index.tsx`
+
+```typescript
+// Add at top level
+let isShuttingDown = false;
+
+async function gracefulShutdown(signal: string): Promise<void> {
+  if (isShuttingDown) return;
+  isShuttingDown = true;
+  
+  console.log(`\nReceived ${signal}, shutting down gracefully...`);
+  
+  try {
+    // Close database
+    closeDb();
+    
+    // Exit cleanly
+    process.exit(0);
+  } catch (error) {
+    console.error('Error during shutdown:', error);
+    process.exit(1);
+  }
+}
+
+// In main function, add before try-catch
+process.on('SIGINT', () => gracefulShutdown('SIGINT'));
+process.on('SIGTERM', () => gracefulShutdown('SIGTERM'));
+
+// Also handle uncaught exceptions
+process.on('uncaughtException', (error) => {
+  console.error('Uncaught Exception:', error);
+  gracefulShutdown('uncaughtException');
+});
+
+process.on('unhandledRejection', (reason, promise) => {
+  console.error('Unhandled Rejection at:', promise, 'reason:', reason);
+  gracefulShutdown('unhandledRejection');
+});
+```
+
+### 3. TKT-012: Add GitHub Token Security
+**File:** `src/services/github.ts`
+
+```typescript
+// Add validation function
+function validateToken(token: string): boolean {
+  // GitHub tokens are typically 40 characters (classic) or start with 'ghp_', 'github_pat_' (fine-grained)
+  const patterns = [
+    /^ghp_[a-zA-Z0-9]{36}$/,        // Personal access tokens
+    /^github_pat_[a-zA-Z0-9]{22}_[a-zA-Z0-9]{59}$/, // Fine-grained tokens
+    /^[a-zA-Z0-9]{40}$/              // Classic tokens
+  ];
+  
+  return patterns.some(pattern => pattern.test(token));
+}
+
+// Update getToken method
+getToken(): string | null {
+  const token = process.env.GITHUB_TOKEN ?? process.env.GH_TOKEN ?? null;
+  
+  if (token && !validateToken(token)) {
+    console.warn('Warning: GITHUB_TOKEN format appears invalid');
+  }
+  
+  return token;
+}
+
+// Add token permission check (optional, requires additional API call)
+async function checkTokenPermissions(token: string): Promise<void> {
+  try {
+    const response = await fetch('https://api.github.com/user', {
+      headers: {
+        'Authorization': `Bearer ${token}`,
+        'Accept': 'application/vnd.github+json'
+      }
+    });
+    
+    if (response.ok) {
+      const scopes = response.headers.get('X-OAuth-Scopes');
+      if (scopes && scopes.includes('delete_repo')) {
+        console.warn('Warning: Token has delete_repo permission which is potentially dangerous');
+      }
+    }
+  } catch {
+    // Silently fail - this is just a warning
+  }
+}
+```
+
+## Phase 2: Core Reliability (Week 2)
+
+### 4. TKT-003: Remove Deprecated Functions
+**Check if batch operations exist:**
+
+```bash
+# Check if operations/batch.ts exists
+ls -la src/operations/
+```
+
+**If batch operations exist:**
+```typescript
+// Update imports in useKeyBindings.ts
+// Replace:
+import { pullAllProjects, pushProjects, fetchAllProjects } from "../git/operations.ts";
+
+// With:
+import { batchPull, batchPush, batchFetch } from "../operations/batch.ts";
+```
+
+**If not, remove deprecation warnings:**
+```typescript
+// In git/operations.ts, remove @deprecated comments
+export async function pullAllProjects(...) {
+  // Remove: @deprecated Use batchPull from operations/batch.ts instead
+}
+```
+
+### 5. TKT-004: Add Retry Logic
+**Create new file:** `src/utils/retry.ts`
+
+```typescript
+export class RetryError extends Error {
+  constructor(message: string, public attempts: number, public lastError: Error) {
+    super(message);
+    this.name = 'RetryError';
+  }
+}
+
+export interface RetryOptions {
+  maxAttempts?: number;
+  initialDelay?: number;
+  maxDelay?: number;
+  backoffFactor?: number;
+  shouldRetry?: (error: Error, attempt: number) => boolean;
+  onRetry?: (error: Error, attempt: number, nextDelay: number) => void;
+}
+
+export async function withRetry<T>(
+  fn: () => Promise<T>,
+  options: RetryOptions = {}
+): Promise<T> {
+  const {
+    maxAttempts = 3,
+    initialDelay = 1000,
+    maxDelay = 30000,
+    backoffFactor = 2,
+    shouldRetry = () => true,
+    onRetry
+  } = options;
+
+  let lastError: Error;
+  
+  for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+    try {
+      return await fn();
+    } catch (error) {
+      lastError = error as Error;
+      
+      if (attempt === maxAttempts || !shouldRetry(lastError, attempt)) {
+        throw new RetryError(
+          `Failed after ${attempt} attempts: ${lastError.message}`,
+          attempt,
+          lastError
+        );
+      }
+      
+      const delay = Math.min(
+        initialDelay * Math.pow(backoffFactor, attempt - 1),
+        maxDelay
+      );
+      
+      onRetry?.(lastError, attempt, delay);
+      
+      await new Promise(resolve => setTimeout(resolve, delay));
+    }
+  }
+  
+  throw lastError!;
+}
+
+// Specific retry logic for GitHub API
+export function shouldRetryGitHubAPI(error: Error, attempt: number): boolean {
+  // Don't retry client errors (4xx) except 429 (rate limit)
+  if (error instanceof GitHubAPIError) {
+    if (error.status === 429) return true; // Rate limit - retry
+    if (error.status >= 400 && error.status < 500) return false; // Client error - don't retry
+    if (error.status >= 500) return true; // Server error - retry
+  }
+  
+  // Network errors - retry
+  if (error.message.includes('network') || error.message.includes('fetch')) {
+    return true;
+  }
+  
+  return attempt < 3; // Default retry for unknown errors
+}
+```
+
+**Update GitHub service to use retry:**
+```typescript
+// In services/github.ts
+async function githubFetch<T>(endpoint: string, token: string): Promise<T> {
+  return withRetry(
+    async () => {
+      if (!token) {
+        throw new GitHubAPIError("GITHUB_TOKEN not set", 401);
+      }
+
+      const url = endpoint.startsWith("https://")
+        ? endpoint
+        : `https://api.github.com${endpoint}`;
+
+      const response = await fetch(url, {
+        headers: {
+          Accept: "application/vnd.github+json",
+          Authorization: `Bearer ${token}`,
+          "X-GitHub-Api-Version": "2022-11-28",
+        },
+      });
+
+      if (!response.ok) {
+        const error = await response.json().catch(() => ({}));
+        throw new GitHubAPIError(
+          `GitHub API error: ${response.statusText}`,
+          response.status,
+          error
+        );
+      }
+
+      return response.json() as Promise<T>;
+    },
+    {
+      shouldRetry: shouldRetryGitHubAPI,
+      onRetry: (error, attempt, delay) => {
+        console.log(`GitHub API retry attempt ${attempt} after ${delay}ms: ${error.message}`);
+      }
+    }
+  );
+}
+```
+
+### 6. TKT-005: Convert to Async Operations
+**File:** `src/scanner/index.ts`
+
+```typescript
+// Change imports at top
+import { readdir, stat } from 'fs/promises';
+
+// Update scanDirectory function
+async function scanDirectory(
+  dirPath: string,
+  config: GitforestConfig,
+  depth: number,
+  maxDepth: number,
+  foundProjects: Project[],
+  processedPaths: Set<string>
+): Promise<void> {
+  if (depth > maxDepth) return;
+  if (processedPaths.has(dirPath)) return;
+  processedPaths.add(dirPath);
+  if (!existsSync(dirPath)) return;
+
+  const isGit = await isGitRepo(dirPath);
+  
+  if (isGit) {
+    const project = await createProject(dirPath, "git");
+    foundProjects.push(project);
+    
+    if (config.display.showSubmodules) {
+      const submodulePaths = await findSubmodules(dirPath);
+      for (const subPath of submodulePaths) {
+        if (!processedPaths.has(subPath)) {
+          const subProject = await createProject(subPath, "git-submodule");
+          foundProjects.push(subProject);
+          processedPaths.add(subPath);
+        }
+      }
+    }
+    return;
+  }
+
+  const marker = await detectProjectMarker(dirPath);
+  if (marker) {
+    const project = await createProject(dirPath, "non-git", marker);
+    foundProjects.push(project);
+    return;
+  }
+
+  // Use async readdir
+  try {
+    const entries = await readdir(dirPath);
+    
+    for (const entry of entries) {
+      if (shouldIgnore(entry, config.scan.ignore, config.scan.includeHidden)) {
+        continue;
+      }
+      
+      const entryPath = join(dirPath, entry);
+      
+      try {
+        const entryStat = await stat(entryPath);
+        if (entryStat.isDirectory()) {
+          await scanDirectory(
+            entryPath,
+            config,
+            depth + 1,
+            maxDepth,
+            foundProjects,
+            processedPaths
+          );
+        }
+      } catch {
+        // Skip entries we can't stat
+      }
+    }
+  } catch {
+    // Skip directories we can't read
+  }
+}
+```
+
+## Phase 3: Performance & Testing (Week 3)
+
+### 7. TKT-006: Add Rate Limiting
+**Create:** `src/utils/rate-limiter.ts`
+
+```typescript
+export class RateLimiter {
+  private queue: Array<() => void> = [];
+  private running = 0;
+  
+  constructor(private maxConcurrent: number) {}
+  
+  async acquire<T>(fn: () => Promise<T>): Promise<T> {
+    return new Promise<T>((resolve, reject) => {
+      this.queue.push(async () => {
+        try {
+          this.running++;
+          const result = await fn();
+          resolve(result);
+        } catch (error) {
+          reject(error);
+        } finally {
+          this.running--;
+          this.process();
+        }
+      });
+      
+      this.process();
+    });
+  }
+  
+  private process(): void {
+    if (this.running < this.maxConcurrent && this.queue.length > 0) {
+      const next = this.queue.shift();
+      if (next) next();
+    }
+  }
+}
+
+// Batch processor with rate limiting
+export async function processInBatches<T, R>(
+  items: T[],
+  processor: (item: T) => Promise<R>,
+  options: {
+    batchSize?: number;
+    maxConcurrent?: number;
+    onProgress?: (completed: number, total: number) => void;
+  } = {}
+): Promise<R[]> {
+  const { batchSize = 10, maxConcurrent = 5, onProgress } = options;
+  
+  const limiter = new RateLimiter(maxConcurrent);
+  const results: R[] = [];
+  let completed = 0;
+  
+  // Process in batches
+  for (let i = 0; i < items.length; i += batchSize) {
+    const batch = items.slice(i, i + batchSize);
+    
+    const batchResults = await Promise.all(
+      batch.map(item =>
+        limiter.acquire(async () => {
+          const result = await processor(item);
+          completed++;
+          onProgress?.(completed, items.length);
+          return result;
+        })
+      )
+    );
+    
+    results.push(...batchResults);
+  }
+  
+  return results;
+}
+```
+
+**Update git operations:**
+```typescript
+// In git/operations.ts
+import { processInBatches } from '../utils/rate-limiter';
+
+export async function pullAllProjects(
+  projects: Project[],
+  concurrency = 5,
+  onProgress?: (completed: number, total: number) => void
+): Promise<BatchResult> {
+  const start = Date.now();
+  const gitProjects = projects.filter(p => p.type === "git" && p.status?.hasRemote);
+  
+  const results = await processInBatches(
+    gitProjects,
+    p => pullProject(p.path),
+    {
+      batchSize: concurrency,
+      maxConcurrent: concurrency,
+      onProgress
+    }
+  );
+  
+  const successful = results.filter(r => r.success).length;
+  
+  return {
+    total: gitProjects.length,
+    successful,
+    failed: gitProjects.length - successful,
+    results,
+    duration: Date.now() - start,
+  };
+}
+```
+
+### 8. TKT-009: Improve Test Coverage
+**Create comprehensive tests:**
+
+```typescript
+// test/hooks/useKeyBindings.test.tsx
+import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
+import { renderHook, act } from '@testing-library/react';
+import { useKeyBindings } from '../../src/hooks/useKeyBindings';
+import { createMockProject } from '../mocks';
+
+describe('useKeyBindings', () => {
+  let onRefresh: jest.Mock;
+  let mockConfig: any;
+  
+  beforeEach(() => {
+    onRefresh = jest.fn().mockResolvedValue(undefined);
+    mockConfig = {
+      scan: { concurrency: 5 },
+      display: { sortBy: 'status', sortDirection: 'desc' }
+    };
+  });
+  
+  afterEach(() => {
+    jest.clearAllMocks();
+  });
+  
+  test('should handle push operation', async () => {
+    const { result } = renderHook(() => 
+      useKeyBindings({ config: mockConfig, onRefresh })
+    );
+    
+    // Simulate 'p' key press
+    await act(async () => {
+      // Simulate keyboard event
+      const event = new KeyboardEvent('keypress', { key: 'p' });
+      window.dispatchEvent(event);
+    });
+    
+    // Verify push was called
+    expect(onRefresh).toHaveBeenCalled();
+  });
+  
+  test('should handle errors gracefully', async () => {
+    onRefresh.mockRejectedValue(new Error('Push failed'));
+    
+    const { result } = renderHook(() => 
+      useKeyBindings({ config: mockConfig, onRefresh })
+    );
+    
+    // Should not throw
+    await act(async () => {
+      const event = new KeyboardEvent('keypress', { key: 'p' });
+      window.dispatchEvent(event);
+    });
+    
+    // Error should be handled internally
+    expect(onRefresh).toHaveBeenCalled();
+  });
+});
+```
+
+### 9. TKT-010: Add CLI Validation
+**Update argument parser:**
+
+```typescript
+// index.tsx
+function parseArgs(args: string[]): {
+  command: string | null;
+  flags: Record<string, boolean | string>;
+  positional: string[];
+  errors: string[];
+} {
+  const flags: Record<string, boolean | string> = {};
+  const positional: string[] = [];
+  const errors: string[] = [];
+  let command: string | null = null;
+  
+  // Track seen flags to detect duplicates
+  const seenFlags = new Set<string>();
+  
+  for (let i = 0; i < args.length; i++) {
+    const arg = args[i]!;
+    
+    if (arg.startsWith("--")) {
+      const key = arg.slice(2);
+      
+      // Check for unknown flags
+      const knownFlags = [
+        'init', 'help', 'json', 'verbose', 'filter', 'target', 'public',
+        'local', 'combined', 'https', 'h', 'v', 'f', 't'
+      ];
+      
+      if (!knownFlags.includes(key) && !knownFlags.includes(key.split('=')[0])) {
+        errors.push(`Unknown flag: --${key}`);
+        continue;
+      }
+      
+      // Check for duplicates
+      if (seenFlags.has(key)) {
+        errors.push(`Duplicate flag: --${key}`);
+        continue;
+      }
+      seenFlags.add(key);
+      
+      // Parse value
+      if (key === "filter" || key === "target") {
+        const nextArg = args[i + 1];
+        if (nextArg && !nextArg.startsWith("-")) {
+          flags[key] = nextArg;
+          i++;
+        } else {
+          errors.push(`Flag --${key} requires a value`);
+        }
+      } else {
+        flags[key] = true;
+      }
+    } else if (arg.startsWith("-") && arg.length === 2) {
+      // Handle short flags similarly...
+    }
+  }
+  
+  return { command, flags, positional, errors };
+}
+```
+
+## Phase 4: Documentation & Polish (Week 4)
+
+### 10. TKT-007: Extract Constants
+**Create:** `src/constants.ts`
+
+```typescript
+// UI Constants
+export const UI = {
+  HEADER_HEIGHT: 6,
+  MIN_TERMINAL_HEIGHT: 24,
+  PROJECT_LIST_PADDING: 2,
+  STATUS_BAR_HEIGHT: 3,
+  FILTER_BAR_HEIGHT: 2,
+} as const;
+
+// API Constants
+export const API = {
+  GITHUB_PAGE_SIZE: 100,
+  GITHUB_MAX_RETRIES: 3,
+  GITHUB_INITIAL_RETRY_DELAY: 1000,
+  GITHUB_MAX_RETRY_DELAY: 30000,
+} as const;
+
+// Scanner Constants
+export const SCANNER = {
+  DEFAULT_MAX_DEPTH: 2,
+  DEFAULT_CONCURRENCY: 5,
+  PROJECT_ID_LENGTH: 12,
+  CACHE_TTL_SECONDS: 300,
+} as const;
+
+// Git Constants
+export const GIT = {
+  DEFAULT_REMOTE: 'origin',
+  DEFAULT_BRANCH: 'main',
+  SUBMODULE_STATUS_REGEX: /^([-\+U ])([a-f0-9]{40}) (.+?)(?: \((.+)\))?$/,
+} as const;
+```
+
+### 11. TKT-008: Add JSDoc
+**Example for public API:**
+
+```typescript
+/**
+ * Scan all configured directories for projects
+ * @param config - The gitforest configuration object
+ * @param options - Optional scanning options
+ * @param options.forceRefresh - Force a fresh scan even if cache is valid
+ * @param options.onProgress - Callback for scan progress updates
+ * @returns Promise resolving to array of found projects
+ * @throws {Error} If scanning fails due to permission errors
+ * 
+ * @example
+ * ```typescript
+ * const projects = await scanWithCache(config, {
+ *   forceRefresh: true,
+ *   onProgress: (scanned, found) => console.log(`Found ${found} projects in ${scanned} directories`)
+ * });
+ * ```
+ */
+export async function scanWithCache(
+  config: GitforestConfig,
+  options: {
+    forceRefresh?: boolean;
+    onProgress?: (scanned: number, found: number) => void;
+  } = {}
+): Promise<Project[]> {
+  // implementation...
+}
+```
+
+### 12. TKT-011: Create Documentation
+**Create:** `CONTRIBUTING.md`
+
+```markdown
+# Contributing to Gitforest
+
+## Development Setup
+
+1. Install Bun runtime
+2. Clone the repository
+3. Run `bun install`
+4. Run `bun test` to verify setup
+
+## Architecture
+
+### Project Structure
+- `src/cli/` - Command-line interface
+- `src/components/` - React components for TUI
+- `src/hooks/` - React hooks
+- `src/state/` - State management
+- `src/services/` - External service integrations
+- `src/git/` - Git operations
+- `src/scanner/` - Directory scanning
+- `src/db/` - SQLite database
+
+### Key Patterns
+- Service abstraction for testability
+- Async/await for all I/O operations
+- Error boundaries for graceful failure
+- Rate limiting for batch operations
+
+## Testing
+- Unit tests: `bun test`
+- Integration tests: `bun test test/integration/`
+- Coverage: `bun test --coverage`
+
+## Code Style
+- TypeScript strict mode
+- No unused variables or imports
+- JSDoc for public APIs
+- Error handling with specific error types
+```
+
+## Testing All Changes
+
+Create a comprehensive test script:
+
+```bash
+#!/bin/bash
+# test-all.sh
+
+echo "Running comprehensive tests..."
+
+# Unit tests
+echo "1. Running unit tests..."
+bun test --coverage
+
+# Integration tests
+echo "2. Running integration tests..."
+bun test test/integration/
+
+# Type checking
+echo "3. Type checking..."
+bun run typecheck
+
+# Linting
+echo "4. Linting..."
+bun run lint
+
+# Test CLI commands
+echo "5. Testing CLI commands..."
+bun run index.tsx --help
+bun run index.tsx cache status
+bun run index.tsx list --json
+
+# Test with sample config
+echo "6. Testing with sample configuration..."
+cp config/gitforest.example.yaml /tmp/test-config.yaml
+bun run index.tsx --init
+
+# Performance test
+echo "7. Performance test..."
+time bun run index.tsx list
+
+echo "All tests completed!"
+```
+
+## Final Verification Checklist
+
+- [ ] All database operations use proper async/await
+- [ ] Graceful shutdown handles all signals
+- [ ] No deprecated functions in use
+- [ ] GitHub API has retry logic
+- [ ] File operations are async
+- [ ] Batch operations have rate limiting
+- [ ] Test coverage >80%
+- [ ] CLI validates all inputs
+- [ ] Constants extracted from code
+- [ ] Public APIs documented
+- [ ] Documentation complete
+- [ ] No TypeScript errors or warnings
+- [ ] All tests pass
+- [ ] Performance acceptable
+
+## Deployment Considerations
+
+1. **Backward Compatibility**: Ensure config format hasn't changed
+2. **Migration**: No database schema changes needed
+3. **Rollback**: Keep previous version tagged
+4. **Monitoring**: Add metrics for error rates and performance
+5. **Documentation**: Update README with new features
+
+This completes the comprehensive improvement plan for the gitforest codebase.