motia 0.5.11-beta.120-742949 → 0.5.11-beta.120-433270

package/dist/cjs/cloud/build/builders/python/add-package-to-archive.js

@@ -6,7 +6,6 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.addPackageToArchive = void 0;
 const fs_1 = __importDefault(require("fs"));
 const path_1 = __importDefault(require("path"));
-const colors_1 = __importDefault(require("colors"));
 const shouldIgnore = (filePath) => {
     const ignorePatterns = [/\.pyc$/, /\.egg$/, /__pycache__/, /\.dist-info$/];
     return ignorePatterns.some((pattern) => pattern.test(filePath));
@@ -31,26 +30,24 @@ const addDirectoryToArchive = async (archive, baseDir, dirPath) => {
         .filter(Boolean));
 };
 const addPackageToArchive = async (archive, sitePackagesDir, packageName) => {
-    // First try the package name as is
-    let fullPath = path_1.default.join(sitePackagesDir, packageName);
-    // If not found, try with .py extension
-    if (!fs_1.default.existsSync(fullPath)) {
-        const pyPath = path_1.default.join(sitePackagesDir, `${packageName}.py`);
-        if (fs_1.default.existsSync(pyPath)) {
-            fullPath = pyPath;
+    const packageNameVariations = [packageName, `${packageName}.py`];
+    // Iterate over all possible package name variations
+    for (const pkg of packageNameVariations) {
+        let fullPath = path_1.default.join(sitePackagesDir, pkg);
+        if (!fs_1.default.existsSync(fullPath)) {
+            // If not found, try next package name variation
+            continue;
         }
-    }
-    if (!fs_1.default.existsSync(fullPath)) {
-        console.log(colors_1.default.yellow(`Warning: Package not found in site-packages: ${packageName}`));
+        const stat = fs_1.default.statSync(fullPath);
+        if (stat.isDirectory()) {
+            await addDirectoryToArchive(archive, sitePackagesDir, fullPath);
+        }
+        else {
+            const relativePath = path_1.default.relative(sitePackagesDir, fullPath);
+            archive.append(fs_1.default.createReadStream(fullPath), relativePath);
+        }
+        // package added successfully
         return;
     }
-    const stat = fs_1.default.statSync(fullPath);
-    if (stat.isDirectory()) {
-        await addDirectoryToArchive(archive, sitePackagesDir, fullPath);
-    }
-    else {
-        const relativePath = path_1.default.relative(sitePackagesDir, fullPath);
-        archive.append(fs_1.default.createReadStream(fullPath), relativePath);
-    }
 };
 exports.addPackageToArchive = addPackageToArchive;

package/dist/cjs/cloud/build/builders/python/index.js

@@ -23,60 +23,33 @@ class PythonBuilder {
         const normalizedEntrypointPath = entrypointPath.replace(/[.]step.py$/, '_step.py');
         const sitePackagesDir = `${process.env.PYTHON_SITE_PACKAGES}-lambda`;
         // Get Python builder response
-        const { packages, local_files } = await this.getPythonBuilderData(step);
+        const { packages, files } = await this.getPythonBuilderData(step);
         // Add main file to archive
         if (!fs_1.default.existsSync(step.filePath)) {
             throw new Error(`Source file not found: ${step.filePath}`);
         }
         archive.append(fs_1.default.createReadStream(step.filePath), path_1.default.relative(this.builder.projectDir, normalizedEntrypointPath));
-        // Add local Python files to archive
-        if (local_files && local_files.length > 0) {
-            local_files.forEach((localFile) => {
-                const fullPath = path_1.default.join(this.builder.projectDir, localFile);
-                if (fs_1.default.existsSync(fullPath)) {
-                    archive.append(fs_1.default.createReadStream(fullPath), localFile);
-                }
-            });
+        files.forEach((file) => archive.append(fs_1.default.createReadStream(file), path_1.default.relative(this.builder.projectDir, file)));
+        if (packages.length > 0) {
+            await Promise.all(packages.map((pkg) => (0, add_package_to_archive_1.addPackageToArchive)(archive, sitePackagesDir, pkg.name)));
+            this.listener.onBuildProgress(step, `Added ${packages.length} packages to archive`);
         }
-        await Promise.all(packages.map(async (packageName) => (0, add_package_to_archive_1.addPackageToArchive)(archive, sitePackagesDir, packageName)));
         return normalizedEntrypointPath;
     }
     async build(step) {
         const entrypointPath = step.filePath.replace(this.builder.projectDir, '');
         const bundlePath = path_1.default.join('python', entrypointPath.replace(/(.*)\.py$/, '$1.zip'));
-        const normalizedEntrypointPath = entrypointPath.replace(/[.]step.py$/, '_step.py');
         const outfile = path_1.default.join(constants_1.distDir, bundlePath);
         try {
             // Create output directory
             fs_1.default.mkdirSync(path_1.default.dirname(outfile), { recursive: true });
             this.listener.onBuildStart(step);
-            // Get Python builder response
-            const { packages, local_files } = await this.getPythonBuilderData(step);
+            // Create the step zip archive
             const stepArchiver = new archiver_1.Archiver(outfile);
+            // Build the step
             const stepPath = await this.buildStep(step, stepArchiver);
-            // Add main file to archive
-            if (!fs_1.default.existsSync(step.filePath)) {
-                throw new Error(`Source file not found: ${step.filePath}`);
-            }
-            stepArchiver.append(fs_1.default.createReadStream(step.filePath), path_1.default.relative(this.builder.projectDir, normalizedEntrypointPath));
-            // Add all imported files to archive
-            this.listener.onBuildProgress(step, 'Adding imported files to archive...');
-            const sitePackagesDir = `${process.env.PYTHON_SITE_PACKAGES}-lambda`;
-            // Add local Python files to archive
-            if (local_files && local_files.length > 0) {
-                local_files.forEach((localFile) => {
-                    const fullPath = path_1.default.join(this.builder.projectDir, localFile);
-                    if (fs_1.default.existsSync(fullPath)) {
-                        stepArchiver.append(fs_1.default.createReadStream(fullPath), localFile);
-                    }
-                });
-                this.listener.onBuildProgress(step, `Added ${local_files.length} local Python files to archive`);
-            }
+            // Add static files to the archive
             (0, include_static_files_1.includeStaticFiles)([step], this.builder, stepArchiver);
-            if (packages.length > 0) {
-                await Promise.all(packages.map(async (packageName) => (0, add_package_to_archive_1.addPackageToArchive)(stepArchiver, sitePackagesDir, packageName)));
-                this.listener.onBuildProgress(step, `Added ${packages.length} packages to archive`);
-            }
             // Finalize the archive and wait for completion
             const size = await stepArchiver.finalize();
             this.builder.registerStep({ entrypointPath: stepPath, bundlePath, step, type: 'python' });
@@ -119,7 +92,7 @@ class PythonBuilder {
     }
     async getPythonBuilderData(step) {
         return new Promise((resolve, reject) => {
-            const child = (0, child_process_1.spawn)('python', [path_1.default.join(__dirname, 'python-builder.py'), step.filePath], {
+            const child = (0, child_process_1.spawn)('python', [path_1.default.join(__dirname, 'python-builder.py'), this.builder.projectDir, step.filePath], {
                 cwd: this.builder.projectDir,
                 stdio: [undefined, undefined, 'pipe', 'ipc'],
             });

package/dist/cjs/cloud/build/builders/python/python-builder.py

@@ -1,313 +1,39 @@
 import os
 import sys
 import json
-import importlib.util
 import traceback
-import site
-import builtins
-import ast
-import importlib.metadata
-import subprocess
-import re
-from typing import Set, List, Tuple, Optional, Dict, Any
-from pathlib import Path
-from functools import lru_cache
 
-NODEIPCFD = int(os.environ["NODE_CHANNEL_FD"])
+from trace_packages import trace_packages
+from trace_project_files import trace_project_files
 
-# Cache for built-in modules to avoid repeated checks
-_builtin_modules_cache: Set[str] = set()
-
-@lru_cache(maxsize=1024)
-def is_valid_package_name(name: str) -> bool:
-    """Check if a name is a valid package name."""
-    if not name or name.startswith('_'):
-        return False
-    
-    # Skip common special cases
-    invalid_names = {'__main__', 'module', 'cython_runtime', 'builtins'}
-    return name not in invalid_names
-
-@lru_cache(maxsize=1024)
-def get_package_name(module_name: str) -> str:
-    """Get the top-level package name from a module name."""
-    return module_name.split('.')[0]
-
-@lru_cache(maxsize=1024)
-def clean_package_name(package_name: str) -> str:
-    """Clean package name by removing version specifiers and other metadata."""
-    # Remove version specifiers and conditions using regex
-    package_name = re.sub(r'[<>=~!;].*$', '', package_name)
-    # Remove any remaining whitespace and convert underscores to hyphens
-    return package_name.strip().replace('_', '-')
-
-@lru_cache(maxsize=1024)
-def extract_base_package_name(dependency_spec: str) -> str:
-    """
-    Extract the base package name from a complex dependency specification.
-    Handles cases like:
-    - 'package (>=1.2.1,<2.0.0)'
-    - 'package[extra] (>=1.2.1)'
-    - 'package ; extra == "vlm"'
-    - 'package (>=1.2.1) ; sys_platform == "darwin"'
-    """
-    # First, remove any conditions after semicolon
-    base_spec = dependency_spec.split(';')[0].strip()
-    
-    # Extract the package name before any version specifiers or extras
-    match = re.match(r'^([a-zA-Z0-9_.-]+)(?:\[[^\]]+\])?(?:\s*\([^)]*\))?$', base_spec)
-    
-    return clean_package_name(match.group(1) if match else base_spec)
-
-@lru_cache(maxsize=1024)
-def is_package_installed(package_name: str) -> bool:
-    """Check if a package is installed in the current environment."""
-    try:
-        # Try both hyphenated and non-hyphenated versions
-        try:
-            importlib.metadata.distribution(package_name)
-            return True
-        except importlib.metadata.PackageNotFoundError:
-            # Try with hyphens replaced by underscores
-            alt_name = package_name.replace('-', '_')
-            if alt_name != package_name:
-                importlib.metadata.distribution(alt_name)
-                return True
-            return False
-    except importlib.metadata.PackageNotFoundError:
-        return False
-
-@lru_cache(maxsize=1024)
-def is_builtin_module(module_name: str) -> bool:
-    """Check if a module is a Python built-in module."""
-    if module_name in _builtin_modules_cache:
-        return True
-    
-    # First check if it's a known built-in module name
-    builtin_modules = {
-        'os', 'sys', 'json', 'math', 'random', 'datetime', 'time', 'urllib', 'http', 
-        'pathlib', 're', 'collections', 'itertools', 'functools', 'operator', 'typing',
-        'io', 'csv', 'xml', 'html', 'email', 'base64', 'hashlib', 'hmac', 'uuid',
-        'pickle', 'sqlite3', 'logging', 'unittest', 'argparse', 'configparser',
-        'tempfile', 'shutil', 'glob', 'fnmatch', 'subprocess', 'threading', 'queue',
-        'multiprocessing', 'concurrent', 'asyncio', 'socket', 'ssl', 'gzip', 'zipfile',
-        'tarfile', 'zlib', 'bz2', 'lzma', 'struct', 'array', 'ctypes', 'mmap',
-        'weakref', 'gc', 'inspect', 'dis', 'ast', 'token', 'tokenize', 'keyword',
-        'builtins', '__main__', 'site', 'sysconfig', 'platform', 'warnings'
-    }
-    
-    if module_name in builtin_modules:
-        _builtin_modules_cache.add(module_name)
-        return True
-        
-    try:
-        module = importlib.import_module(module_name)
-        
-        # Built-in modules either have no __file__ attribute or their file is in the standard library
-        if not hasattr(module, '__file__'):
-            _builtin_modules_cache.add(module_name)
-            return True
-            
-        # Get the standard library path
-        stdlib_path = os.path.dirname(os.__file__)
-        
-        # Check if the module's file is in the standard library
-        is_builtin = module.__file__ and module.__file__.startswith(stdlib_path)
-        if is_builtin:
-            _builtin_modules_cache.add(module_name)
-        return is_builtin
-    except ImportError:
-        # If we can't import it, assume it's not a built-in module
-        # This handles local modules that aren't in the current Python path
-        return False
-
-def get_direct_imports(file_path: str) -> Set[str]:
-    """Extract direct imports from a Python file using AST parsing."""
-    direct_imports = set()
-    
-    try:
-        with open(file_path, 'r') as f:
-            content = f.read()
-        
-        tree = ast.parse(content)
-        for node in ast.walk(tree):
-            if isinstance(node, ast.Import):
-                for name in node.names:
-                    base_pkg = name.name.split('.')[0]
-                    if is_valid_package_name(base_pkg) and not is_builtin_module(base_pkg):
-                        direct_imports.add(base_pkg)
-            elif isinstance(node, ast.ImportFrom):
-                if node.module:
-                    base_pkg = node.module.split('.')[0]
-                    if is_valid_package_name(base_pkg) and not is_builtin_module(base_pkg):
-                        direct_imports.add(base_pkg)
-    except Exception as e:
-        print(f"Warning: Could not parse imports from {file_path}: {str(e)}")
-    
-    return direct_imports
-
-def get_all_python_files(project_root: str) -> List[str]:
-    """Get all Python files in the project."""
-    python_files = []
-    for root, dirs, files in os.walk(project_root):
-        # Skip common directories
-        dirs[:] = [d for d in dirs if not d.startswith('.') and d not in 
-                  {'__pycache__', 'node_modules', 'dist', 'build', 'venv'}]
-        
-        for file in files:
-            if file.endswith('.py') and not file.startswith('.'):
-                full_path = os.path.join(root, file)
-                relative_path = os.path.relpath(full_path, project_root)
-                python_files.append(relative_path)
-    
-    return python_files
-
-def get_imports_from_file(file_path: str) -> Set[str]:
-    """Get all import module names from a Python file."""
-    imports = set()
-    
-    try:
-        with open(file_path, 'r') as f:
-            content = f.read()
-        
-        tree = ast.parse(content)
-        for node in ast.walk(tree):
-            if isinstance(node, ast.Import):
-                for name in node.names:
-                    imports.add(name.name)
-            elif isinstance(node, ast.ImportFrom):
-                if node.module:
-                    imports.add(node.module)
-    except Exception as e:
-        print(f"Warning: Could not parse imports from {file_path}: {str(e)}")
-    
-    return imports
-
-def get_local_files_for_entry(entry_file: str) -> List[str]:
-    """Get local Python files that are imported by the entry file."""
-    # Find project root
-    project_root = os.path.dirname(entry_file)
-    while project_root != os.path.dirname(project_root):
-        if any(os.path.exists(os.path.join(project_root, f)) 
-               for f in ['package.json', 'requirements.txt']):
-            break
-        project_root = os.path.dirname(project_root)
-    
-    # Get all Python files in the project
-    all_python_files = get_all_python_files(project_root)
-    
-    # Get imports from the entry file
-    imports = get_imports_from_file(entry_file)
-    
-    # Check which imports match local Python files
-    local_files = []
-    for import_name in imports:
-        for py_file in all_python_files:
-            # Convert file path to module name (e.g., 'utils/example.py' -> 'utils.example')
-            module_name = py_file.replace(os.sep, '.').replace('.py', '')
-            if import_name == module_name:
-                local_files.append(py_file)
-    
-    return sorted(local_files)
-
-def trace_imports(entry_file: str) -> List[str]:
-    """Find all imported Python packages from entry file and its local imports."""
-    entry_file = os.path.abspath(entry_file)
-    
-    # Get local files that are imported
-    local_files = get_local_files_for_entry(entry_file)
-    
-    # Get project root
-    project_root = os.path.dirname(entry_file)
-    while project_root != os.path.dirname(project_root):
-        if any(os.path.exists(os.path.join(project_root, f)) 
-               for f in ['package.json', 'requirements.txt']):
-            break
-        project_root = os.path.dirname(project_root)
-    
-    # Get imports from entry file and local files
-    all_packages = set()
-    processed_packages = set()
-    files_to_process = [entry_file] + [os.path.join(project_root, f) for f in local_files]
-    
-    for python_file in files_to_process:
-        if os.path.exists(python_file):
-            direct_imports = get_direct_imports(python_file)
-            for package_name in direct_imports:
-                if is_valid_package_name(package_name) and not is_builtin_module(package_name):
-                    all_packages.add(package_name)
-                    # Get all dependencies including sub-dependencies
-                    all_packages.update(get_package_dependencies(package_name, processed_packages))
-    
-    return sorted(list(all_packages))
-
-@lru_cache(maxsize=1024)
-def is_optional_dependency(req: str) -> bool:
-    """Check if a dependency is an optional dependency."""
-    return '[' in req or 'extra ==' in req
-
-def get_package_dependencies(package_name: str, processed: Set[str] = None) -> Set[str]:
-    """Get all dependencies (including sub-dependencies) for a given package."""
-    if processed is None:
-        processed = set()
-    
-    if package_name in processed or is_builtin_module(package_name):
-        return set()
-    
-    processed.add(package_name)
-    all_dependencies = set()
-    
-    try:
-        # Try to get the distribution
-        try:
-            dist = importlib.metadata.distribution(package_name)
-        except importlib.metadata.PackageNotFoundError:
-            print(f'Warning: Package {package_name} not found')
-            return all_dependencies
-
-        # Filter out optional dependencies
-        sub_dependencies = list(filter(lambda dep: not is_optional_dependency(dep), dist.requires or []))
-        
-        # Get direct dependencies
-        for req in sub_dependencies:                
-            base_pkg = extract_base_package_name(req)
-            
-            if base_pkg and base_pkg not in processed:
-                # Try both hyphenated and non-hyphenated versions
-                for dep_name in [base_pkg, base_pkg.replace('-', '_'), base_pkg.replace('_', '-')]:
-                    try:
-                        importlib.import_module(dep_name)
-                        all_dependencies.add(dep_name)
-                        # Recursively get sub-dependencies
-                        all_dependencies.update(get_package_dependencies(dep_name, processed))
-                        break
-                    except ImportError:
-                        continue
-            
-    except Exception as e:
-        print(f"Warning: Error processing {package_name}: {str(e)}")
-    
-    return all_dependencies
+NODEIPCFD = int(os.environ.get("NODE_CHANNEL_FD", 0))
 
 def main() -> None:
     """Main entry point for the script."""
-    if len(sys.argv) != 2:
-        print("Usage: python python-builder.py <entry_file>", file=sys.stderr)
-        sys.exit(1)
+    if len(sys.argv) != 3:
+        print("Usage: python python-builder.py <project_dir> <entry_file>")
+        sys.exit(2)
 
-    entry_file = sys.argv[1]
-    try:
-        packages = trace_imports(entry_file)
-        local_files = get_local_files_for_entry(entry_file)
-        
+    project_dir = os.path.abspath(sys.argv[1])
+    entry_file = os.path.abspath(sys.argv[2])
 
+    try:
+        # Find project dependencies
+        packages = trace_packages(project_dir, entry_file)
+        files = trace_project_files(project_dir, entry_file)
         
+        # Prepare output
         output = {
             'packages': packages,
-            'local_files': local_files
+            'files': files,
         }
+        
+        # Output as JSON
         bytes_message = (json.dumps(output) + '\n').encode('utf-8')
-        os.write(NODEIPCFD, bytes_message)
+        if NODEIPCFD > 0:
+            os.write(NODEIPCFD, bytes_message)
+        else:
+            print(bytes_message)
         sys.exit(0)
     except Exception as e:
         print(f"Error: {str(e)}", file=sys.stderr)

package/dist/cjs/cloud/build/builders/python/trace_packages.py

@@ -0,0 +1,212 @@
+import os
+import sys
+import importlib.util
+import ast
+import importlib.metadata
+import re
+from typing import Set, List
+from functools import lru_cache
+
+# Cache for built-in modules to avoid repeated checks
+_builtin_modules_cache: Set[str] = set()
+
+@lru_cache(maxsize=1024)
+def is_valid_package_name(name: str) -> bool:
+    """Check if a name is a valid package name."""
+    if not name or name.startswith('_'):
+        return False
+    
+    # Skip common special cases
+    invalid_names = {'__main__', 'module', 'cython_runtime', 'builtins'}
+    return name not in invalid_names
+
+@lru_cache(maxsize=1024)
+def get_package_name(module_name: str) -> str:
+    """Get the top-level package name from a module name."""
+    return module_name.split('.')[0]
+
+@lru_cache(maxsize=1024)
+def clean_package_name(package_name: str) -> str:
+    """Clean package name by removing version specifiers and other metadata."""
+    # Remove version specifiers and conditions using regex
+    package_name = re.sub(r'[<>=~!;].*$', '', package_name)
+    # Remove any remaining whitespace and convert underscores to hyphens
+    return package_name.strip().replace('_', '-')
+
+@lru_cache(maxsize=1024)
+def extract_base_package_name(dependency_spec: str) -> str:
+    """
+    Extract the base package name from a complex dependency specification.
+    Handles cases like:
+    - 'package (>=1.2.1,<2.0.0)'
+    - 'package[extra] (>=1.2.1)'
+    - 'package ; extra == "vlm"'
+    - 'package (>=1.2.1) ; sys_platform == "darwin"'
+    """
+    # First, remove any conditions after semicolon
+    base_spec = dependency_spec.split(';')[0].strip()
+    
+    # Extract the package name before any version specifiers or extras
+    match = re.match(r'^([a-zA-Z0-9_.-]+)(?:\[[^\]]+\])?(?:\s*\([^)]*\))?$', base_spec)
+    
+    return clean_package_name(match.group(1) if match else base_spec)
+
+@lru_cache(maxsize=1024)
+def is_package_installed(package_name: str) -> bool:
+    """Check if a package is installed in the current environment."""
+    try:
+        # Try both hyphenated and non-hyphenated versions
+        try:
+            importlib.metadata.distribution(package_name)
+            return True
+        except importlib.metadata.PackageNotFoundError:
+            # Try with hyphens replaced by underscores
+            alt_name = package_name.replace('-', '_')
+            if alt_name != package_name:
+                importlib.metadata.distribution(alt_name)
+                return True
+            return False
+    except importlib.metadata.PackageNotFoundError:
+        return False
+
+@lru_cache(maxsize=1024)
+def is_builtin_module(module_name: str) -> bool:
+    """Check if a module is a Python built-in module."""
+    if module_name in _builtin_modules_cache:
+        return True
+        
+    try:
+        module = importlib.import_module(module_name)
+        
+        # Built-in modules either have no __file__ attribute or their file is in the standard library
+        if not hasattr(module, '__file__'):
+            _builtin_modules_cache.add(module_name)
+            return True
+            
+        # Get the standard library path
+        stdlib_path = os.path.dirname(os.__file__)
+        
+        # Check if the module's file is in the standard library
+        is_builtin = module.__file__ and module.__file__.startswith(stdlib_path)
+        if is_builtin:
+            _builtin_modules_cache.add(module_name)
+        return is_builtin
+    except ImportError:
+        return False
+
+def is_local_import(package_name: str, project_dir: str) -> bool:
+    """Check if a package is a local import within the project directory."""
+    try:
+        # Try to import the module
+        module = importlib.import_module(package_name)
+        
+        # If the module has no __file__ attribute, it's not a local file
+        if not hasattr(module, '__file__'):
+            return False
+            
+        # Check if the module's file is within the project directory
+        module_path = os.path.abspath(module.__file__)
+        
+        # Check if the module path is not in a site-packages or dist-packages directory
+        return 'site-packages' not in module_path and 'dist-packages' not in module_path
+        
+    except ImportError:
+        # If we can't import it, it means it's a local import
+        return True
+
+def get_direct_imports(project_dir: str, file_path: str) -> Set[str]:
+    """Extract direct imports from a Python file using AST parsing."""
+    direct_imports = set()
+    
+    try:
+        with open(file_path, 'r') as f:
+            content = f.read()
+        
+        tree = ast.parse(content)
+        for node in ast.walk(tree):
+            if isinstance(node, ast.Import):
+                for name in node.names:
+                    base_pkg = name.name.split('.')[0]
+                    if is_valid_package_name(base_pkg) and not is_builtin_module(base_pkg):
+                        # Check if this is a local import
+                        if not is_local_import(base_pkg, project_dir):
+                            direct_imports.add(base_pkg)
+            elif isinstance(node, ast.ImportFrom):
+                if node.module:
+                    base_pkg = node.module.split('.')[0]
+                    if is_valid_package_name(base_pkg) and not is_builtin_module(base_pkg):
+                        # Check if this is a local import
+                        if not is_local_import(base_pkg, project_dir):
+                            direct_imports.add(base_pkg)
+    except Exception as e:
+        print(f"Warning: Could not parse imports from {file_path}: {str(e)}")
+    
+    return direct_imports
+
+@lru_cache(maxsize=1024)
+def is_optional_dependency(req: str) -> bool:
+    """Check if a dependency is an optional dependency."""
+    return '[' in req or 'extra ==' in req
+
+def get_package_dependencies(package_name: str, processed: Set[str] = None) -> Set[str]:
+    """Get all dependencies (including sub-dependencies) for a given package."""
+    if processed is None:
+        processed = set()
+    
+    if package_name in processed:
+        return set()
+    
+    processed.add(package_name)
+    all_dependencies = set()
+    
+    try:
+        # Try to get the distribution
+        try:
+            dist = importlib.metadata.distribution(package_name)
+        except importlib.metadata.PackageNotFoundError:
+            print(f'Warning: Package {package_name} not found')
+            return all_dependencies
+
+        # Filter out optional dependencies
+        sub_dependencies = list(filter(lambda dep: not is_optional_dependency(dep), dist.requires or []))
+        
+        # Get direct dependencies
+        for req in sub_dependencies:                
+            base_pkg = extract_base_package_name(req)
+            
+            if base_pkg and base_pkg not in processed:
+                # Try both hyphenated and non-hyphenated versions
+                for dep_name in [base_pkg, base_pkg.replace('-', '_'), base_pkg.replace('_', '-')]:
+                    try:
+                        importlib.import_module(dep_name)
+                        all_dependencies.add(dep_name)
+                        # Recursively get sub-dependencies
+                        all_dependencies.update(get_package_dependencies(dep_name, processed))
+                        break
+                    except ImportError:
+                        continue
+            
+    except Exception as e:
+        print(f"Warning: Error processing {package_name}: {str(e)}")
+    
+    return all_dependencies
+
+def trace_packages(project_dir: str, entry_file: str) -> List[str]:
+    """Find all imported Python packages and files starting from an entry file."""
+    
+    # Get direct imports from the entry file
+    direct_imports = get_direct_imports(project_dir, entry_file)
+    
+    # Initialize lists to track packages
+    all_packages = []
+    processed_packages = set()
+    
+    # Process each direct import and its dependencies
+    for package_name in direct_imports:
+        all_packages.append({ 'name': package_name, 'is_direct_import': True })
+        # Get all dependencies including sub-dependencies
+        dependencies = get_package_dependencies(package_name, processed_packages)
+        all_packages.extend([{ 'name': dep, 'is_direct_import': False } for dep in dependencies])        
+    
+    # Filter out built-in packages
+    return all_packages