@principal-ai/codebase-composition 0.1.0

package/src/providers/index.ts

@@ -0,0 +1,17 @@
+// Version control providers
+export { GitVersionControlProvider } from './GitVersionControlProvider';
+export type { FileSystemAdapter, GitAdapter, ShellAdapter } from './GitVersionControlProvider';
+
+// Package management providers
+export {
+  PackageManagerApiProvider,
+  type PackageManager,
+  type PackageVersionInfo,
+  type VersionCheckResult,
+  type VulnerabilityInfo,
+  type VulnerabilityCheckResult,
+  type LicenseInfo,
+  type LicenseCheckResult,
+  type DependencyCheckProgress,
+  type BatchCheckOptions,
+} from './PackageManagerApiProvider';

package/src/services/FilesystemService.ts

@@ -0,0 +1,530 @@
+import ignore from 'ignore';
+
+import { FileTree, FileInfo, DirectoryInfo } from '@principal-ai/repository-abstraction';
+import { FileSystemAdapter } from '../providers';
+import { FileSystemFilterLayer } from '../types';
+
+// Use path operations that work in browser
+const path = {
+  basename: (p: string) => p.split('/').pop() || '',
+  join: (...parts: string[]) => {
+    const joined = parts.filter(p => p).join('/');
+    return joined.replace(/\/+/g, '/');
+  },
+  relative: (from: string, to: string) => {
+    if (!from || from === '.') {
+      return to;
+    }
+    if (to.startsWith(from + '/')) {
+      return to.slice(from.length + 1);
+    }
+    if (to === from) {
+      return '.';
+    }
+    return to;
+  },
+  dirname: (p: string) => {
+    const lastSlash = p.lastIndexOf('/');
+    return lastSlash > 0 ? p.slice(0, lastSlash) : '';
+  },
+  extname: (p: string) => {
+    const lastDot = p.lastIndexOf('.');
+    return lastDot > 0 ? p.slice(lastDot) : '';
+  },
+};
+
+interface FilterStats {
+  totalFiltered: number;
+  filteredByLayer: Record<string, number>;
+  examples: Array<{ path: string; filteredBy: string }>;
+  filteredByDirectory: Record<string, { count: number; reasons: Set<string> }>;
+}
+
+export class FilesystemService {
+  private filterStats: FilterStats = {
+    totalFiltered: 0,
+    filteredByLayer: {},
+    examples: [],
+    filteredByDirectory: {},
+  };
+
+  constructor(private adapter?: FileSystemAdapter) {}
+
+  /**
+   * Build file tree using the provided adapter with efficient filtering
+   */
+  async buildFileSystemTreeFromPath(
+    directoryPath: string,
+    filters?: FileSystemFilterLayer[],
+    options?: { enableInverseFiltering?: boolean } | object,
+  ): Promise<
+    FileTree & {
+      filterStats?: FilterStats;
+      appliedFilters?: FileSystemFilterLayer[];
+      inverseMatches?: Array<{ path: string; matchedFilters: string[] }>;
+    }
+  > {
+    console.time('[FilesystemService] Total build time');
+
+    // Reset filter stats
+    this.filterStats = {
+      totalFiltered: 0,
+      filteredByLayer: {},
+      examples: [],
+      filteredByDirectory: {},
+    };
+
+    const allFilters = filters || [];
+    // Handle both old and new options format for backward compatibility
+    const enableInverseFiltering =
+      (options as { enableInverseFiltering?: boolean })?.enableInverseFiltering || false;
+
+    // Handle case when no adapter is provided
+    if (!this.adapter) {
+      // Return empty tree structure for backward compatibility
+      const rootDir = {
+        path: directoryPath,
+        name: directoryPath.split('/').pop() || directoryPath,
+        children: [],
+        fileCount: 0,
+        totalSize: 0,
+        depth: 0,
+        relativePath: '.',
+      };
+
+      return {
+        sha: '',
+        root: rootDir,
+        allFiles: [],
+        allDirectories: [rootDir],
+        stats: {
+          totalFiles: 0,
+          totalDirectories: 1,
+          totalSize: 0,
+          maxDepth: 0,
+          buildingTypeDistribution: {},
+          directoryTypeDistribution: {},
+          combinedTypeDistribution: {},
+        },
+        filterStats: this.filterStats,
+        appliedFilters: allFilters,
+        inverseMatches: enableInverseFiltering ? [] : undefined,
+      };
+    }
+
+    console.time('[FilesystemService] Build file tree');
+
+    // Use the adapter's optimized implementation
+    const patterns: string[] = [];
+    let sourceDir: string | undefined;
+
+    // Convert filters to patterns for the adapter
+    for (const filter of allFilters) {
+      if (filter.enabled && filter.filterData?.excludedPatterns) {
+        patterns.push(...filter.filterData.excludedPatterns.map(p => p.pattern));
+        if (filter.filterData.sourceDirectory) {
+          sourceDir = filter.filterData.sourceDirectory;
+        }
+      }
+    }
+
+    const adapterResult = await this.adapter.buildFilteredFileTree(
+      directoryPath,
+      patterns,
+      sourceDir,
+    );
+    const crawledPaths = adapterResult.paths;
+    const fileStats = adapterResult.stats;
+
+    // Update filter stats
+    const ignoreInstances = this.createIgnoreInstances(allFilters);
+    for (const path of crawledPaths) {
+      if (!this.shouldIncludeFile(path, ignoreInstances, allFilters)) {
+        this.filterStats.totalFiltered++;
+      }
+    }
+
+    console.timeEnd('[FilesystemService] Build file tree');
+    console.log(`[FilesystemService] Found ${crawledPaths.length} paths after filtering`);
+
+    // Convert paths to FileInfo and DirectoryInfo objects
+    console.time('[FilesystemService] Convert to tree structure');
+    const { allFiles, allDirectories, rootDir } = await this.convertPathsToTree(
+      crawledPaths,
+      directoryPath,
+      fileStats,
+    );
+    console.timeEnd('[FilesystemService] Convert to tree structure');
+
+    // Handle inverse filtering for power users
+    let inverseMatches: Array<{ path: string; matchedFilters: string[] }> = [];
+    if (enableInverseFiltering) {
+      inverseMatches = await this.findInverseMatches(directoryPath, ignoreInstances, allFilters);
+    }
+
+    // Calculate stats
+    const totalSize = allFiles.reduce((sum, file) => sum + (file.size || 0), 0);
+    const maxDepth = Math.max(...allDirectories.map(d => d.depth || 0));
+
+    // Update filter layer stats
+    allFilters.forEach(layer => {
+      if (layer.enabled && this.filterStats.filteredByLayer[layer.id]) {
+        if (!layer.filterData.stats) {
+          layer.filterData.stats = {
+            totalFiltered: 0,
+            bytesExcluded: 0,
+          };
+        }
+        layer.filterData.stats.totalFiltered = this.filterStats.filteredByLayer[layer.id];
+      }
+    });
+
+    const treeResult: FileTree & {
+      filterStats?: FilterStats;
+      appliedFilters?: FileSystemFilterLayer[];
+      inverseMatches?: Array<{ path: string; matchedFilters: string[] }>;
+    } = {
+      sha: '',
+      root: rootDir || {
+        path: '.',
+        name: path.basename(directoryPath) || 'root',
+        children: [],
+        fileCount: 0,
+        totalSize: 0,
+        depth: 0,
+        relativePath: '.',
+      },
+      allFiles,
+      allDirectories,
+      stats: {
+        totalFiles: allFiles.length,
+        totalDirectories: allDirectories.length,
+        totalSize,
+        maxDepth,
+        buildingTypeDistribution: {},
+        directoryTypeDistribution: {},
+        combinedTypeDistribution: {},
+      },
+      filterStats: this.filterStats,
+      appliedFilters: allFilters,
+      inverseMatches: inverseMatches,
+    };
+
+    console.timeEnd('[FilesystemService] Total build time');
+    return treeResult;
+  }
+
+  /**
+   * Create optimized ignore instances from filter layers
+   */
+  private createIgnoreInstances(filters: FileSystemFilterLayer[]): Map<
+    string,
+    {
+      ignore: ReturnType<typeof ignore>;
+      sourceDirectory: string;
+      filterId: string;
+      filterName: string;
+    }
+  > {
+    const ignoreInstances = new Map<
+      string,
+      {
+        ignore: ReturnType<typeof ignore>;
+        sourceDirectory: string;
+        filterId: string;
+        filterName: string;
+      }
+    >();
+
+    for (const filter of filters) {
+      if (!filter.enabled || !filter.filterData?.excludedPatterns) continue;
+
+      const sourceDirectory = filter.filterData.sourceDirectory || '';
+      const key = `${filter.id}:${sourceDirectory}`;
+
+      const ig = ignore();
+      const patterns = filter.filterData.excludedPatterns.map(p => p.pattern);
+      ig.add(patterns);
+
+      ignoreInstances.set(key, {
+        ignore: ig,
+        sourceDirectory,
+        filterId: filter.id,
+        filterName: filter.name,
+      });
+    }
+
+    return ignoreInstances;
+  }
+
+  /**
+   * Fast filtering using node-ignore during fdir crawl
+   */
+  private shouldIncludeFile(
+    filePath: string,
+    ignoreInstances: Map<
+      string,
+      {
+        ignore: ReturnType<typeof ignore>;
+        sourceDirectory: string;
+        filterId: string;
+        filterName: string;
+      }
+    >,
+    _filters: FileSystemFilterLayer[],
+  ): boolean {
+    for (const [, { ignore: ig, sourceDirectory, filterId, filterName }] of ignoreInstances) {
+      // Check if this path is within the filter's scope
+      if (sourceDirectory !== '') {
+        if (!filePath.startsWith(sourceDirectory + '/') && filePath !== sourceDirectory) {
+          continue; // Skip this filter - path is outside its scope
+        }
+      }
+
+      // Get the path relative to the filter's source directory for testing
+      let pathToTest = filePath;
+      if (sourceDirectory !== '' && filePath.startsWith(sourceDirectory + '/')) {
+        pathToTest = filePath.slice(sourceDirectory.length + 1);
+      }
+
+      // Skip empty paths and "." which node-ignore doesn't handle
+      if (!pathToTest || pathToTest === '.') {
+        continue;
+      }
+
+      // Use node-ignore for fast pattern matching
+      if (ig.ignores(pathToTest)) {
+        // Track statistics
+        this.filterStats.totalFiltered++;
+        this.filterStats.filteredByLayer[filterId] =
+          (this.filterStats.filteredByLayer[filterId] || 0) + 1;
+
+        // Track by directory
+        const dirPath = path.dirname(filePath) || '.';
+        if (!this.filterStats.filteredByDirectory[dirPath]) {
+          this.filterStats.filteredByDirectory[dirPath] = { count: 0, reasons: new Set() };
+        }
+        this.filterStats.filteredByDirectory[dirPath].count++;
+        this.filterStats.filteredByDirectory[dirPath].reasons.add(filterName);
+
+        return false; // Exclude this file
+      }
+    }
+
+    return true; // Include this file
+  }
+
+  /**
+   * Convert array of paths to FileTree structure
+   */
+  private async convertPathsToTree(
+    paths: string[],
+    rootPath: string,
+    providedStats?: Map<string, { size: number; isDirectory: boolean; lastModified: Date }>,
+  ): Promise<{
+    allFiles: FileInfo[];
+    allDirectories: DirectoryInfo[];
+    rootDir: DirectoryInfo | null;
+  }> {
+    const allFiles: FileInfo[] = [];
+    const allDirectories: DirectoryInfo[] = [];
+    const directoryMap = new Map<string, DirectoryInfo>();
+
+    // Use provided stats or fetch them if adapter supports it
+    const fileStats =
+      providedStats ||
+      new Map<string, { size: number; lastModified: Date; isDirectory: boolean }>();
+
+    if (!providedStats && this.adapter && 'getFileStats' in this.adapter) {
+      // Batch file stats operations for better performance
+      const statPromises = paths.map(async filePath => {
+        try {
+          const fullPath = path.join(rootPath, filePath);
+          const stats = await (
+            this.adapter as FileSystemAdapter & {
+              getFileStats: (
+                path: string,
+              ) => Promise<{ size?: number; lastModified?: Date; isDirectory?: boolean }>;
+            }
+          ).getFileStats(fullPath);
+          if (stats) {
+            fileStats.set(filePath, {
+              size: stats.size || 0,
+              lastModified: stats.lastModified || new Date(),
+              isDirectory: stats.isDirectory || false,
+            });
+          }
+        } catch {
+          // Ignore stat errors, will use defaults
+        }
+      });
+
+      await Promise.all(statPromises);
+    }
+
+    // Create file and directory objects
+    for (const filePath of paths) {
+      const stats = fileStats.get(filePath);
+      const isDirectory = stats?.isDirectory || filePath.endsWith('/');
+
+      if (isDirectory) {
+        const dirInfo: DirectoryInfo = {
+          path: filePath || '.',
+          name: path.basename(filePath) || 'root',
+          children: [],
+          fileCount: 0,
+          totalSize: 0,
+          depth: filePath.split('/').length - 1,
+          relativePath: filePath || '.',
+        };
+
+        allDirectories.push(dirInfo);
+        directoryMap.set(filePath, dirInfo);
+      } else {
+        const fileInfo: FileInfo = {
+          path: filePath,
+          name: path.basename(filePath),
+          extension: path.extname(filePath).toLowerCase(),
+          size: stats?.size || 0,
+          lastModified: stats?.lastModified || new Date(),
+          isDirectory: false,
+          relativePath: filePath,
+        };
+
+        allFiles.push(fileInfo);
+      }
+    }
+
+    // Build hierarchical structure (simplified - just return root)
+    const rootDir = directoryMap.get('.') ||
+      directoryMap.get('') || {
+        path: '.',
+        name: path.basename(rootPath) || 'root',
+        children: [],
+        fileCount: allFiles.length,
+        totalSize: allFiles.reduce((sum, f) => sum + (f.size || 0), 0),
+        depth: 0,
+        relativePath: '.',
+      };
+
+    return { allFiles, allDirectories, rootDir };
+  }
+
+  /**
+   * Find files that would match specific filters - for power user investigation
+   */
+  private async findInverseMatches(
+    projectPath: string,
+    ignoreInstances: Map<
+      string,
+      {
+        ignore: ReturnType<typeof ignore>;
+        sourceDirectory: string;
+        filterId: string;
+        filterName: string;
+      }
+    >,
+    _filters: FileSystemFilterLayer[],
+  ): Promise<Array<{ path: string; matchedFilters: string[] }>> {
+    console.log('[FilesystemService] Running inverse filter analysis...');
+
+    const inverseMatches: Array<{ path: string; matchedFilters: string[] }> = [];
+
+    // Get ALL files without any filtering to get the complete picture
+    if (!this.adapter) {
+      return [];
+    }
+    const unfilteredResult = await this.adapter.buildFilteredFileTree(projectPath);
+    const allPaths = unfilteredResult.paths;
+
+    console.log(`[FilesystemService] Found ${allPaths.length} total paths for inverse analysis`);
+
+    // Test each path against each filter to see what matches
+    for (const filePath of allPaths) {
+      const matchedFilters: string[] = [];
+
+      for (const [, { ignore: ig, sourceDirectory, filterName }] of ignoreInstances) {
+        // Check if this path is within the filter's scope
+        if (sourceDirectory !== '') {
+          if (!filePath.startsWith(sourceDirectory + '/') && filePath !== sourceDirectory) {
+            continue; // Skip this filter - path is outside its scope
+          }
+        }
+
+        // Get the path relative to the filter's source directory for testing
+        let pathToTest = filePath;
+        if (sourceDirectory !== '' && filePath.startsWith(sourceDirectory + '/')) {
+          pathToTest = filePath.slice(sourceDirectory.length + 1);
+        }
+
+        // Skip empty paths and "." which node-ignore doesn't handle
+        if (!pathToTest || pathToTest === '.') {
+          continue;
+        }
+
+        // Check if this filter would match this path
+        if (ig.ignores(pathToTest)) {
+          matchedFilters.push(filterName);
+        }
+      }
+
+      // If any filters matched, record it
+      if (matchedFilters.length > 0) {
+        inverseMatches.push({
+          path: filePath,
+          matchedFilters,
+        });
+      }
+    }
+
+    console.log(
+      `[FilesystemService] Inverse analysis complete: ${inverseMatches.length} files matched filters`,
+    );
+    return inverseMatches;
+  }
+
+  /**
+   * Get the filter statistics from the last build
+   */
+  getFilterStats(): FilterStats {
+    return { ...this.filterStats };
+  }
+
+  /**
+   * Create an empty tree structure - matches original FilesystemService API
+   */
+  private getEmptyTree(directoryPath: string): FileTree & {
+    filterStats?: FilterStats;
+    appliedFilters?: FileSystemFilterLayer[];
+    inverseMatches?: Array<{ path: string; matchedFilters: string[] }>;
+  } {
+    const rootDir: DirectoryInfo = {
+      path: '.',
+      name: path.basename(directoryPath) || 'root',
+      children: [],
+      fileCount: 0,
+      totalSize: 0,
+      depth: 0,
+      relativePath: '.',
+    };
+
+    return {
+      sha: '',
+      root: rootDir,
+      allFiles: [],
+      allDirectories: [rootDir],
+      stats: {
+        totalFiles: 0,
+        totalDirectories: 1,
+        totalSize: 0,
+        maxDepth: 0,
+        buildingTypeDistribution: {},
+        directoryTypeDistribution: {},
+        combinedTypeDistribution: {},
+      },
+      filterStats: this.filterStats,
+      appliedFilters: [],
+      inverseMatches: [],
+    };
+  }
+}

package/src/services/index.ts

@@ -0,0 +1,2 @@
+// File system services
+export { FilesystemService } from './FilesystemService';

package/src/types/file-system.ts

@@ -0,0 +1,11 @@
+/**
+ * File system types for layer modules
+ * Re-exported from core/file-tree to maintain consistency
+ */
+
+// Re-export from the central file-tree module
+export type { FileInfo, DirectoryInfo, FileTree } from '@principal-ai/repository-abstraction';
+
+// Alias FileTree as FileSystemTree for backward compatibility
+// This can be removed in a future version
+export type { FileTree as FileSystemTree } from '@principal-ai/repository-abstraction';

package/src/types/index.ts

@@ -0,0 +1,24 @@
+// Export layer types
+export type {
+  FilePattern,
+  FileSet,
+  BaseLayer,
+  PackageCommand,
+  ConfigFile,
+  PackageLayer,
+  FileSystemFilterLayer,
+  ExtractedType,
+  PackageTypes,
+  TypeDefinitionLayer,
+  TypeExtractor,
+} from './layer-types';
+
+// Export version control types
+export type {
+  VersionControlLayer,
+  VersionControlIgnorePattern,
+  VersionControlProvider,
+} from './version-control-layer';
+
+// Export file system types
+export type { FileInfo, DirectoryInfo, FileTree, FileSystemTree } from './file-system';