motia 0.5.11-beta.119 → 0.5.11-beta.120-433270
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/cloud/build/builders/python/add-package-to-archive.js +16 -19
- package/dist/cjs/cloud/build/builders/python/index.js +10 -18
- package/dist/cjs/cloud/build/builders/python/python-builder.py +22 -204
- package/dist/cjs/cloud/build/builders/python/trace_packages.py +212 -0
- package/dist/cjs/cloud/build/builders/python/trace_project_files.py +41 -0
- package/dist/cjs/create/templates/basic-tutorial/01-api.step.ts.txt +1 -33
- package/dist/cjs/create/templates/basic-tutorial/02-process-food-order.step.ts.txt +6 -30
- package/dist/cjs/create/templates/basic-tutorial/04_new_order_notifications.step.py.txt +6 -1
- package/dist/cjs/create/templates/basic-tutorial/motia-workbench.json +13 -12
- package/dist/cjs/create/templates/generate.js +3 -1
- package/dist/cjs/create/templates/generate.ts +3 -1
- package/dist/cjs/dev-watchers.js +3 -2
- package/dist/cjs/generate-locked-data.js +2 -2
- package/dist/cjs/generate-types.js +2 -2
- package/dist/cjs/utils/install-lambda-python-packages.js +1 -1
- package/dist/cjs/watcher.d.ts +2 -1
- package/dist/cjs/watcher.js +6 -5
- package/dist/esm/cloud/build/builders/python/add-package-to-archive.js +16 -19
- package/dist/esm/cloud/build/builders/python/index.js +10 -18
- package/dist/esm/cloud/build/builders/python/python-builder.py +22 -204
- package/dist/esm/cloud/build/builders/python/trace_packages.py +212 -0
- package/dist/esm/cloud/build/builders/python/trace_project_files.py +41 -0
- package/dist/esm/create/templates/basic-tutorial/01-api.step.ts.txt +1 -33
- package/dist/esm/create/templates/basic-tutorial/02-process-food-order.step.ts.txt +6 -30
- package/dist/esm/create/templates/basic-tutorial/04_new_order_notifications.step.py.txt +6 -1
- package/dist/esm/create/templates/basic-tutorial/motia-workbench.json +13 -12
- package/dist/esm/create/templates/generate.js +3 -1
- package/dist/esm/create/templates/generate.ts +3 -1
- package/dist/esm/dev-watchers.js +3 -2
- package/dist/esm/generate-locked-data.js +2 -2
- package/dist/esm/generate-types.js +2 -2
- package/dist/esm/utils/install-lambda-python-packages.js +1 -1
- package/dist/esm/watcher.d.ts +2 -1
- package/dist/esm/watcher.js +6 -5
- package/dist/types/watcher.d.ts +2 -1
- package/package.json +4 -4
- package/dist/cjs/create/templates/basic-tutorial/00-noop.step.ts.txt +0 -30
- package/dist/esm/create/templates/basic-tutorial/00-noop.step.ts.txt +0 -30
|
@@ -6,7 +6,6 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
6
6
|
exports.addPackageToArchive = void 0;
|
|
7
7
|
const fs_1 = __importDefault(require("fs"));
|
|
8
8
|
const path_1 = __importDefault(require("path"));
|
|
9
|
-
const colors_1 = __importDefault(require("colors"));
|
|
10
9
|
const shouldIgnore = (filePath) => {
|
|
11
10
|
const ignorePatterns = [/\.pyc$/, /\.egg$/, /__pycache__/, /\.dist-info$/];
|
|
12
11
|
return ignorePatterns.some((pattern) => pattern.test(filePath));
|
|
@@ -31,26 +30,24 @@ const addDirectoryToArchive = async (archive, baseDir, dirPath) => {
|
|
|
31
30
|
.filter(Boolean));
|
|
32
31
|
};
|
|
33
32
|
const addPackageToArchive = async (archive, sitePackagesDir, packageName) => {
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
33
|
+
const packageNameVariations = [packageName, `${packageName}.py`];
|
|
34
|
+
// Iterate over all possible package name variations
|
|
35
|
+
for (const pkg of packageNameVariations) {
|
|
36
|
+
let fullPath = path_1.default.join(sitePackagesDir, pkg);
|
|
37
|
+
if (!fs_1.default.existsSync(fullPath)) {
|
|
38
|
+
// If not found, try next package name variation
|
|
39
|
+
continue;
|
|
41
40
|
}
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
41
|
+
const stat = fs_1.default.statSync(fullPath);
|
|
42
|
+
if (stat.isDirectory()) {
|
|
43
|
+
await addDirectoryToArchive(archive, sitePackagesDir, fullPath);
|
|
44
|
+
}
|
|
45
|
+
else {
|
|
46
|
+
const relativePath = path_1.default.relative(sitePackagesDir, fullPath);
|
|
47
|
+
archive.append(fs_1.default.createReadStream(fullPath), relativePath);
|
|
48
|
+
}
|
|
49
|
+
// package added successfully
|
|
45
50
|
return;
|
|
46
51
|
}
|
|
47
|
-
const stat = fs_1.default.statSync(fullPath);
|
|
48
|
-
if (stat.isDirectory()) {
|
|
49
|
-
await addDirectoryToArchive(archive, sitePackagesDir, fullPath);
|
|
50
|
-
}
|
|
51
|
-
else {
|
|
52
|
-
const relativePath = path_1.default.relative(sitePackagesDir, fullPath);
|
|
53
|
-
archive.append(fs_1.default.createReadStream(fullPath), relativePath);
|
|
54
|
-
}
|
|
55
52
|
};
|
|
56
53
|
exports.addPackageToArchive = addPackageToArchive;
|
|
@@ -23,41 +23,33 @@ class PythonBuilder {
|
|
|
23
23
|
const normalizedEntrypointPath = entrypointPath.replace(/[.]step.py$/, '_step.py');
|
|
24
24
|
const sitePackagesDir = `${process.env.PYTHON_SITE_PACKAGES}-lambda`;
|
|
25
25
|
// Get Python builder response
|
|
26
|
-
const { packages } = await this.getPythonBuilderData(step);
|
|
26
|
+
const { packages, files } = await this.getPythonBuilderData(step);
|
|
27
27
|
// Add main file to archive
|
|
28
28
|
if (!fs_1.default.existsSync(step.filePath)) {
|
|
29
29
|
throw new Error(`Source file not found: ${step.filePath}`);
|
|
30
30
|
}
|
|
31
31
|
archive.append(fs_1.default.createReadStream(step.filePath), path_1.default.relative(this.builder.projectDir, normalizedEntrypointPath));
|
|
32
|
-
|
|
32
|
+
files.forEach((file) => archive.append(fs_1.default.createReadStream(file), path_1.default.relative(this.builder.projectDir, file)));
|
|
33
|
+
if (packages.length > 0) {
|
|
34
|
+
await Promise.all(packages.map((pkg) => (0, add_package_to_archive_1.addPackageToArchive)(archive, sitePackagesDir, pkg.name)));
|
|
35
|
+
this.listener.onBuildProgress(step, `Added ${packages.length} packages to archive`);
|
|
36
|
+
}
|
|
33
37
|
return normalizedEntrypointPath;
|
|
34
38
|
}
|
|
35
39
|
async build(step) {
|
|
36
40
|
const entrypointPath = step.filePath.replace(this.builder.projectDir, '');
|
|
37
41
|
const bundlePath = path_1.default.join('python', entrypointPath.replace(/(.*)\.py$/, '$1.zip'));
|
|
38
|
-
const normalizedEntrypointPath = entrypointPath.replace(/[.]step.py$/, '_step.py');
|
|
39
42
|
const outfile = path_1.default.join(constants_1.distDir, bundlePath);
|
|
40
43
|
try {
|
|
41
44
|
// Create output directory
|
|
42
45
|
fs_1.default.mkdirSync(path_1.default.dirname(outfile), { recursive: true });
|
|
43
46
|
this.listener.onBuildStart(step);
|
|
44
|
-
//
|
|
45
|
-
const { packages } = await this.getPythonBuilderData(step);
|
|
47
|
+
// Create the step zip archive
|
|
46
48
|
const stepArchiver = new archiver_1.Archiver(outfile);
|
|
49
|
+
// Build the step
|
|
47
50
|
const stepPath = await this.buildStep(step, stepArchiver);
|
|
48
|
-
// Add
|
|
49
|
-
if (!fs_1.default.existsSync(step.filePath)) {
|
|
50
|
-
throw new Error(`Source file not found: ${step.filePath}`);
|
|
51
|
-
}
|
|
52
|
-
stepArchiver.append(fs_1.default.createReadStream(step.filePath), path_1.default.relative(this.builder.projectDir, normalizedEntrypointPath));
|
|
53
|
-
// Add all imported files to archive
|
|
54
|
-
this.listener.onBuildProgress(step, 'Adding imported files to archive...');
|
|
55
|
-
const sitePackagesDir = `${process.env.PYTHON_SITE_PACKAGES}-lambda`;
|
|
51
|
+
// Add static files to the archive
|
|
56
52
|
(0, include_static_files_1.includeStaticFiles)([step], this.builder, stepArchiver);
|
|
57
|
-
if (packages.length > 0) {
|
|
58
|
-
await Promise.all(packages.map(async (packageName) => (0, add_package_to_archive_1.addPackageToArchive)(stepArchiver, sitePackagesDir, packageName)));
|
|
59
|
-
this.listener.onBuildProgress(step, `Added ${packages.length} packages to archive`);
|
|
60
|
-
}
|
|
61
53
|
// Finalize the archive and wait for completion
|
|
62
54
|
const size = await stepArchiver.finalize();
|
|
63
55
|
this.builder.registerStep({ entrypointPath: stepPath, bundlePath, step, type: 'python' });
|
|
@@ -100,7 +92,7 @@ class PythonBuilder {
|
|
|
100
92
|
}
|
|
101
93
|
async getPythonBuilderData(step) {
|
|
102
94
|
return new Promise((resolve, reject) => {
|
|
103
|
-
const child = (0, child_process_1.spawn)('python', [path_1.default.join(__dirname, 'python-builder.py'), step.filePath], {
|
|
95
|
+
const child = (0, child_process_1.spawn)('python', [path_1.default.join(__dirname, 'python-builder.py'), this.builder.projectDir, step.filePath], {
|
|
104
96
|
cwd: this.builder.projectDir,
|
|
105
97
|
stdio: [undefined, undefined, 'pipe', 'ipc'],
|
|
106
98
|
});
|
|
@@ -1,221 +1,39 @@
|
|
|
1
1
|
import os
|
|
2
2
|
import sys
|
|
3
3
|
import json
|
|
4
|
-
import importlib.util
|
|
5
4
|
import traceback
|
|
6
|
-
import site
|
|
7
|
-
import builtins
|
|
8
|
-
import ast
|
|
9
|
-
import importlib.metadata
|
|
10
|
-
import subprocess
|
|
11
|
-
import re
|
|
12
|
-
from typing import Set, List, Tuple, Optional, Dict, Any
|
|
13
|
-
from pathlib import Path
|
|
14
|
-
from functools import lru_cache
|
|
15
5
|
|
|
16
|
-
|
|
6
|
+
from trace_packages import trace_packages
|
|
7
|
+
from trace_project_files import trace_project_files
|
|
17
8
|
|
|
18
|
-
|
|
19
|
-
_builtin_modules_cache: Set[str] = set()
|
|
20
|
-
|
|
21
|
-
@lru_cache(maxsize=1024)
|
|
22
|
-
def is_valid_package_name(name: str) -> bool:
|
|
23
|
-
"""Check if a name is a valid package name."""
|
|
24
|
-
if not name or name.startswith('_'):
|
|
25
|
-
return False
|
|
26
|
-
|
|
27
|
-
# Skip common special cases
|
|
28
|
-
invalid_names = {'__main__', 'module', 'cython_runtime', 'builtins'}
|
|
29
|
-
return name not in invalid_names
|
|
30
|
-
|
|
31
|
-
@lru_cache(maxsize=1024)
|
|
32
|
-
def get_package_name(module_name: str) -> str:
|
|
33
|
-
"""Get the top-level package name from a module name."""
|
|
34
|
-
return module_name.split('.')[0]
|
|
35
|
-
|
|
36
|
-
@lru_cache(maxsize=1024)
|
|
37
|
-
def clean_package_name(package_name: str) -> str:
|
|
38
|
-
"""Clean package name by removing version specifiers and other metadata."""
|
|
39
|
-
# Remove version specifiers and conditions using regex
|
|
40
|
-
package_name = re.sub(r'[<>=~!;].*$', '', package_name)
|
|
41
|
-
# Remove any remaining whitespace and convert underscores to hyphens
|
|
42
|
-
return package_name.strip().replace('_', '-')
|
|
43
|
-
|
|
44
|
-
@lru_cache(maxsize=1024)
|
|
45
|
-
def extract_base_package_name(dependency_spec: str) -> str:
|
|
46
|
-
"""
|
|
47
|
-
Extract the base package name from a complex dependency specification.
|
|
48
|
-
Handles cases like:
|
|
49
|
-
- 'package (>=1.2.1,<2.0.0)'
|
|
50
|
-
- 'package[extra] (>=1.2.1)'
|
|
51
|
-
- 'package ; extra == "vlm"'
|
|
52
|
-
- 'package (>=1.2.1) ; sys_platform == "darwin"'
|
|
53
|
-
"""
|
|
54
|
-
# First, remove any conditions after semicolon
|
|
55
|
-
base_spec = dependency_spec.split(';')[0].strip()
|
|
56
|
-
|
|
57
|
-
# Extract the package name before any version specifiers or extras
|
|
58
|
-
match = re.match(r'^([a-zA-Z0-9_.-]+)(?:\[[^\]]+\])?(?:\s*\([^)]*\))?$', base_spec)
|
|
59
|
-
|
|
60
|
-
return clean_package_name(match.group(1) if match else base_spec)
|
|
61
|
-
|
|
62
|
-
@lru_cache(maxsize=1024)
|
|
63
|
-
def is_package_installed(package_name: str) -> bool:
|
|
64
|
-
"""Check if a package is installed in the current environment."""
|
|
65
|
-
try:
|
|
66
|
-
# Try both hyphenated and non-hyphenated versions
|
|
67
|
-
try:
|
|
68
|
-
importlib.metadata.distribution(package_name)
|
|
69
|
-
return True
|
|
70
|
-
except importlib.metadata.PackageNotFoundError:
|
|
71
|
-
# Try with hyphens replaced by underscores
|
|
72
|
-
alt_name = package_name.replace('-', '_')
|
|
73
|
-
if alt_name != package_name:
|
|
74
|
-
importlib.metadata.distribution(alt_name)
|
|
75
|
-
return True
|
|
76
|
-
return False
|
|
77
|
-
except importlib.metadata.PackageNotFoundError:
|
|
78
|
-
return False
|
|
79
|
-
|
|
80
|
-
@lru_cache(maxsize=1024)
|
|
81
|
-
def is_builtin_module(module_name: str) -> bool:
|
|
82
|
-
"""Check if a module is a Python built-in module."""
|
|
83
|
-
if module_name in _builtin_modules_cache:
|
|
84
|
-
return True
|
|
85
|
-
|
|
86
|
-
try:
|
|
87
|
-
module = importlib.import_module(module_name)
|
|
88
|
-
|
|
89
|
-
# Built-in modules either have no __file__ attribute or their file is in the standard library
|
|
90
|
-
if not hasattr(module, '__file__'):
|
|
91
|
-
_builtin_modules_cache.add(module_name)
|
|
92
|
-
return True
|
|
93
|
-
|
|
94
|
-
# Get the standard library path
|
|
95
|
-
stdlib_path = os.path.dirname(os.__file__)
|
|
96
|
-
|
|
97
|
-
# Check if the module's file is in the standard library
|
|
98
|
-
is_builtin = module.__file__ and module.__file__.startswith(stdlib_path)
|
|
99
|
-
if is_builtin:
|
|
100
|
-
_builtin_modules_cache.add(module_name)
|
|
101
|
-
return is_builtin
|
|
102
|
-
except ImportError:
|
|
103
|
-
return False
|
|
104
|
-
|
|
105
|
-
def get_direct_imports(file_path: str) -> Set[str]:
|
|
106
|
-
"""Extract direct imports from a Python file using AST parsing."""
|
|
107
|
-
direct_imports = set()
|
|
108
|
-
|
|
109
|
-
try:
|
|
110
|
-
with open(file_path, 'r') as f:
|
|
111
|
-
content = f.read()
|
|
112
|
-
|
|
113
|
-
tree = ast.parse(content)
|
|
114
|
-
for node in ast.walk(tree):
|
|
115
|
-
if isinstance(node, ast.Import):
|
|
116
|
-
for name in node.names:
|
|
117
|
-
base_pkg = name.name.split('.')[0]
|
|
118
|
-
if is_valid_package_name(base_pkg) and not is_builtin_module(base_pkg):
|
|
119
|
-
direct_imports.add(base_pkg)
|
|
120
|
-
elif isinstance(node, ast.ImportFrom):
|
|
121
|
-
if node.module:
|
|
122
|
-
base_pkg = node.module.split('.')[0]
|
|
123
|
-
if is_valid_package_name(base_pkg) and not is_builtin_module(base_pkg):
|
|
124
|
-
direct_imports.add(base_pkg)
|
|
125
|
-
except Exception as e:
|
|
126
|
-
print(f"Warning: Could not parse imports from {file_path}: {str(e)}")
|
|
127
|
-
|
|
128
|
-
return direct_imports
|
|
129
|
-
|
|
130
|
-
@lru_cache(maxsize=1024)
|
|
131
|
-
def is_optional_dependency(req: str) -> bool:
|
|
132
|
-
"""Check if a dependency is an optional dependency."""
|
|
133
|
-
return '[' in req or 'extra ==' in req
|
|
134
|
-
|
|
135
|
-
def get_package_dependencies(package_name: str, processed: Set[str] = None) -> Set[str]:
|
|
136
|
-
"""Get all dependencies (including sub-dependencies) for a given package."""
|
|
137
|
-
if processed is None:
|
|
138
|
-
processed = set()
|
|
139
|
-
|
|
140
|
-
if package_name in processed or is_builtin_module(package_name):
|
|
141
|
-
return set()
|
|
142
|
-
|
|
143
|
-
processed.add(package_name)
|
|
144
|
-
all_dependencies = set()
|
|
145
|
-
|
|
146
|
-
try:
|
|
147
|
-
# Try to get the distribution
|
|
148
|
-
try:
|
|
149
|
-
dist = importlib.metadata.distribution(package_name)
|
|
150
|
-
except importlib.metadata.PackageNotFoundError:
|
|
151
|
-
print(f'Warning: Package {package_name} not found')
|
|
152
|
-
return all_dependencies
|
|
153
|
-
|
|
154
|
-
# Filter out optional dependencies
|
|
155
|
-
sub_dependencies = list(filter(lambda dep: not is_optional_dependency(dep), dist.requires or []))
|
|
156
|
-
|
|
157
|
-
# Get direct dependencies
|
|
158
|
-
for req in sub_dependencies:
|
|
159
|
-
base_pkg = extract_base_package_name(req)
|
|
160
|
-
|
|
161
|
-
if base_pkg and base_pkg not in processed:
|
|
162
|
-
# Try both hyphenated and non-hyphenated versions
|
|
163
|
-
for dep_name in [base_pkg, base_pkg.replace('-', '_'), base_pkg.replace('_', '-')]:
|
|
164
|
-
try:
|
|
165
|
-
importlib.import_module(dep_name)
|
|
166
|
-
all_dependencies.add(dep_name)
|
|
167
|
-
# Recursively get sub-dependencies
|
|
168
|
-
all_dependencies.update(get_package_dependencies(dep_name, processed))
|
|
169
|
-
break
|
|
170
|
-
except ImportError:
|
|
171
|
-
continue
|
|
172
|
-
|
|
173
|
-
except Exception as e:
|
|
174
|
-
print(f"Warning: Error processing {package_name}: {str(e)}")
|
|
175
|
-
|
|
176
|
-
return all_dependencies
|
|
177
|
-
|
|
178
|
-
def trace_imports(entry_file: str) -> List[str]:
|
|
179
|
-
"""Find all imported Python packages and files starting from an entry file."""
|
|
180
|
-
entry_file = os.path.abspath(entry_file)
|
|
181
|
-
module_dir = os.path.dirname(entry_file)
|
|
182
|
-
|
|
183
|
-
if module_dir not in sys.path:
|
|
184
|
-
sys.path.insert(0, module_dir)
|
|
185
|
-
|
|
186
|
-
# Get direct imports from the entry file
|
|
187
|
-
direct_imports = get_direct_imports(entry_file)
|
|
188
|
-
|
|
189
|
-
# Initialize sets to track packages
|
|
190
|
-
all_packages = set()
|
|
191
|
-
processed_packages = set()
|
|
192
|
-
|
|
193
|
-
# Process each direct import and its dependencies
|
|
194
|
-
for package_name in direct_imports:
|
|
195
|
-
if is_valid_package_name(package_name):
|
|
196
|
-
all_packages.add(package_name)
|
|
197
|
-
# Get all dependencies including sub-dependencies
|
|
198
|
-
all_packages.update(get_package_dependencies(package_name, processed_packages))
|
|
199
|
-
|
|
200
|
-
# Filter out built-in packages
|
|
201
|
-
non_builtin_packages = {pkg for pkg in all_packages if not is_builtin_module(pkg)}
|
|
202
|
-
|
|
203
|
-
return sorted(list(non_builtin_packages))
|
|
9
|
+
NODEIPCFD = int(os.environ.get("NODE_CHANNEL_FD", 0))
|
|
204
10
|
|
|
205
11
|
def main() -> None:
|
|
206
12
|
"""Main entry point for the script."""
|
|
207
|
-
if len(sys.argv) !=
|
|
208
|
-
print("Usage: python python-builder.py <entry_file>"
|
|
209
|
-
sys.exit(
|
|
13
|
+
if len(sys.argv) != 3:
|
|
14
|
+
print("Usage: python python-builder.py <project_dir> <entry_file>")
|
|
15
|
+
sys.exit(2)
|
|
16
|
+
|
|
17
|
+
project_dir = os.path.abspath(sys.argv[1])
|
|
18
|
+
entry_file = os.path.abspath(sys.argv[2])
|
|
210
19
|
|
|
211
|
-
entry_file = sys.argv[1]
|
|
212
20
|
try:
|
|
213
|
-
|
|
21
|
+
# Find project dependencies
|
|
22
|
+
packages = trace_packages(project_dir, entry_file)
|
|
23
|
+
files = trace_project_files(project_dir, entry_file)
|
|
24
|
+
|
|
25
|
+
# Prepare output
|
|
214
26
|
output = {
|
|
215
|
-
'packages': packages
|
|
27
|
+
'packages': packages,
|
|
28
|
+
'files': files,
|
|
216
29
|
}
|
|
30
|
+
|
|
31
|
+
# Output as JSON
|
|
217
32
|
bytes_message = (json.dumps(output) + '\n').encode('utf-8')
|
|
218
|
-
|
|
33
|
+
if NODEIPCFD > 0:
|
|
34
|
+
os.write(NODEIPCFD, bytes_message)
|
|
35
|
+
else:
|
|
36
|
+
print(bytes_message)
|
|
219
37
|
sys.exit(0)
|
|
220
38
|
except Exception as e:
|
|
221
39
|
print(f"Error: {str(e)}", file=sys.stderr)
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import sys
|
|
3
|
+
import importlib.util
|
|
4
|
+
import ast
|
|
5
|
+
import importlib.metadata
|
|
6
|
+
import re
|
|
7
|
+
from typing import Set, List
|
|
8
|
+
from functools import lru_cache
|
|
9
|
+
|
|
10
|
+
# Cache for built-in modules to avoid repeated checks
|
|
11
|
+
_builtin_modules_cache: Set[str] = set()
|
|
12
|
+
|
|
13
|
+
@lru_cache(maxsize=1024)
|
|
14
|
+
def is_valid_package_name(name: str) -> bool:
|
|
15
|
+
"""Check if a name is a valid package name."""
|
|
16
|
+
if not name or name.startswith('_'):
|
|
17
|
+
return False
|
|
18
|
+
|
|
19
|
+
# Skip common special cases
|
|
20
|
+
invalid_names = {'__main__', 'module', 'cython_runtime', 'builtins'}
|
|
21
|
+
return name not in invalid_names
|
|
22
|
+
|
|
23
|
+
@lru_cache(maxsize=1024)
|
|
24
|
+
def get_package_name(module_name: str) -> str:
|
|
25
|
+
"""Get the top-level package name from a module name."""
|
|
26
|
+
return module_name.split('.')[0]
|
|
27
|
+
|
|
28
|
+
@lru_cache(maxsize=1024)
|
|
29
|
+
def clean_package_name(package_name: str) -> str:
|
|
30
|
+
"""Clean package name by removing version specifiers and other metadata."""
|
|
31
|
+
# Remove version specifiers and conditions using regex
|
|
32
|
+
package_name = re.sub(r'[<>=~!;].*$', '', package_name)
|
|
33
|
+
# Remove any remaining whitespace and convert underscores to hyphens
|
|
34
|
+
return package_name.strip().replace('_', '-')
|
|
35
|
+
|
|
36
|
+
@lru_cache(maxsize=1024)
|
|
37
|
+
def extract_base_package_name(dependency_spec: str) -> str:
|
|
38
|
+
"""
|
|
39
|
+
Extract the base package name from a complex dependency specification.
|
|
40
|
+
Handles cases like:
|
|
41
|
+
- 'package (>=1.2.1,<2.0.0)'
|
|
42
|
+
- 'package[extra] (>=1.2.1)'
|
|
43
|
+
- 'package ; extra == "vlm"'
|
|
44
|
+
- 'package (>=1.2.1) ; sys_platform == "darwin"'
|
|
45
|
+
"""
|
|
46
|
+
# First, remove any conditions after semicolon
|
|
47
|
+
base_spec = dependency_spec.split(';')[0].strip()
|
|
48
|
+
|
|
49
|
+
# Extract the package name before any version specifiers or extras
|
|
50
|
+
match = re.match(r'^([a-zA-Z0-9_.-]+)(?:\[[^\]]+\])?(?:\s*\([^)]*\))?$', base_spec)
|
|
51
|
+
|
|
52
|
+
return clean_package_name(match.group(1) if match else base_spec)
|
|
53
|
+
|
|
54
|
+
@lru_cache(maxsize=1024)
|
|
55
|
+
def is_package_installed(package_name: str) -> bool:
|
|
56
|
+
"""Check if a package is installed in the current environment."""
|
|
57
|
+
try:
|
|
58
|
+
# Try both hyphenated and non-hyphenated versions
|
|
59
|
+
try:
|
|
60
|
+
importlib.metadata.distribution(package_name)
|
|
61
|
+
return True
|
|
62
|
+
except importlib.metadata.PackageNotFoundError:
|
|
63
|
+
# Try with hyphens replaced by underscores
|
|
64
|
+
alt_name = package_name.replace('-', '_')
|
|
65
|
+
if alt_name != package_name:
|
|
66
|
+
importlib.metadata.distribution(alt_name)
|
|
67
|
+
return True
|
|
68
|
+
return False
|
|
69
|
+
except importlib.metadata.PackageNotFoundError:
|
|
70
|
+
return False
|
|
71
|
+
|
|
72
|
+
@lru_cache(maxsize=1024)
|
|
73
|
+
def is_builtin_module(module_name: str) -> bool:
|
|
74
|
+
"""Check if a module is a Python built-in module."""
|
|
75
|
+
if module_name in _builtin_modules_cache:
|
|
76
|
+
return True
|
|
77
|
+
|
|
78
|
+
try:
|
|
79
|
+
module = importlib.import_module(module_name)
|
|
80
|
+
|
|
81
|
+
# Built-in modules either have no __file__ attribute or their file is in the standard library
|
|
82
|
+
if not hasattr(module, '__file__'):
|
|
83
|
+
_builtin_modules_cache.add(module_name)
|
|
84
|
+
return True
|
|
85
|
+
|
|
86
|
+
# Get the standard library path
|
|
87
|
+
stdlib_path = os.path.dirname(os.__file__)
|
|
88
|
+
|
|
89
|
+
# Check if the module's file is in the standard library
|
|
90
|
+
is_builtin = module.__file__ and module.__file__.startswith(stdlib_path)
|
|
91
|
+
if is_builtin:
|
|
92
|
+
_builtin_modules_cache.add(module_name)
|
|
93
|
+
return is_builtin
|
|
94
|
+
except ImportError:
|
|
95
|
+
return False
|
|
96
|
+
|
|
97
|
+
def is_local_import(package_name: str, project_dir: str) -> bool:
|
|
98
|
+
"""Check if a package is a local import within the project directory."""
|
|
99
|
+
try:
|
|
100
|
+
# Try to import the module
|
|
101
|
+
module = importlib.import_module(package_name)
|
|
102
|
+
|
|
103
|
+
# If the module has no __file__ attribute, it's not a local file
|
|
104
|
+
if not hasattr(module, '__file__'):
|
|
105
|
+
return False
|
|
106
|
+
|
|
107
|
+
# Check if the module's file is within the project directory
|
|
108
|
+
module_path = os.path.abspath(module.__file__)
|
|
109
|
+
|
|
110
|
+
# Check if the module path is not in a site-packages or dist-packages directory
|
|
111
|
+
return 'site-packages' not in module_path and 'dist-packages' not in module_path
|
|
112
|
+
|
|
113
|
+
except ImportError:
|
|
114
|
+
# If we can't import it, it means it's a local import
|
|
115
|
+
return True
|
|
116
|
+
|
|
117
|
+
def get_direct_imports(project_dir: str, file_path: str) -> Set[str]:
|
|
118
|
+
"""Extract direct imports from a Python file using AST parsing."""
|
|
119
|
+
direct_imports = set()
|
|
120
|
+
|
|
121
|
+
try:
|
|
122
|
+
with open(file_path, 'r') as f:
|
|
123
|
+
content = f.read()
|
|
124
|
+
|
|
125
|
+
tree = ast.parse(content)
|
|
126
|
+
for node in ast.walk(tree):
|
|
127
|
+
if isinstance(node, ast.Import):
|
|
128
|
+
for name in node.names:
|
|
129
|
+
base_pkg = name.name.split('.')[0]
|
|
130
|
+
if is_valid_package_name(base_pkg) and not is_builtin_module(base_pkg):
|
|
131
|
+
# Check if this is a local import
|
|
132
|
+
if not is_local_import(base_pkg, project_dir):
|
|
133
|
+
direct_imports.add(base_pkg)
|
|
134
|
+
elif isinstance(node, ast.ImportFrom):
|
|
135
|
+
if node.module:
|
|
136
|
+
base_pkg = node.module.split('.')[0]
|
|
137
|
+
if is_valid_package_name(base_pkg) and not is_builtin_module(base_pkg):
|
|
138
|
+
# Check if this is a local import
|
|
139
|
+
if not is_local_import(base_pkg, project_dir):
|
|
140
|
+
direct_imports.add(base_pkg)
|
|
141
|
+
except Exception as e:
|
|
142
|
+
print(f"Warning: Could not parse imports from {file_path}: {str(e)}")
|
|
143
|
+
|
|
144
|
+
return direct_imports
|
|
145
|
+
|
|
146
|
+
@lru_cache(maxsize=1024)
|
|
147
|
+
def is_optional_dependency(req: str) -> bool:
|
|
148
|
+
"""Check if a dependency is an optional dependency."""
|
|
149
|
+
return '[' in req or 'extra ==' in req
|
|
150
|
+
|
|
151
|
+
def get_package_dependencies(package_name: str, processed: Set[str] = None) -> Set[str]:
|
|
152
|
+
"""Get all dependencies (including sub-dependencies) for a given package."""
|
|
153
|
+
if processed is None:
|
|
154
|
+
processed = set()
|
|
155
|
+
|
|
156
|
+
if package_name in processed:
|
|
157
|
+
return set()
|
|
158
|
+
|
|
159
|
+
processed.add(package_name)
|
|
160
|
+
all_dependencies = set()
|
|
161
|
+
|
|
162
|
+
try:
|
|
163
|
+
# Try to get the distribution
|
|
164
|
+
try:
|
|
165
|
+
dist = importlib.metadata.distribution(package_name)
|
|
166
|
+
except importlib.metadata.PackageNotFoundError:
|
|
167
|
+
print(f'Warning: Package {package_name} not found')
|
|
168
|
+
return all_dependencies
|
|
169
|
+
|
|
170
|
+
# Filter out optional dependencies
|
|
171
|
+
sub_dependencies = list(filter(lambda dep: not is_optional_dependency(dep), dist.requires or []))
|
|
172
|
+
|
|
173
|
+
# Get direct dependencies
|
|
174
|
+
for req in sub_dependencies:
|
|
175
|
+
base_pkg = extract_base_package_name(req)
|
|
176
|
+
|
|
177
|
+
if base_pkg and base_pkg not in processed:
|
|
178
|
+
# Try both hyphenated and non-hyphenated versions
|
|
179
|
+
for dep_name in [base_pkg, base_pkg.replace('-', '_'), base_pkg.replace('_', '-')]:
|
|
180
|
+
try:
|
|
181
|
+
importlib.import_module(dep_name)
|
|
182
|
+
all_dependencies.add(dep_name)
|
|
183
|
+
# Recursively get sub-dependencies
|
|
184
|
+
all_dependencies.update(get_package_dependencies(dep_name, processed))
|
|
185
|
+
break
|
|
186
|
+
except ImportError:
|
|
187
|
+
continue
|
|
188
|
+
|
|
189
|
+
except Exception as e:
|
|
190
|
+
print(f"Warning: Error processing {package_name}: {str(e)}")
|
|
191
|
+
|
|
192
|
+
return all_dependencies
|
|
193
|
+
|
|
194
|
+
def trace_packages(project_dir: str, entry_file: str) -> List[str]:
|
|
195
|
+
"""Find all imported Python packages and files starting from an entry file."""
|
|
196
|
+
|
|
197
|
+
# Get direct imports from the entry file
|
|
198
|
+
direct_imports = get_direct_imports(project_dir, entry_file)
|
|
199
|
+
|
|
200
|
+
# Initialize lists to track packages
|
|
201
|
+
all_packages = []
|
|
202
|
+
processed_packages = set()
|
|
203
|
+
|
|
204
|
+
# Process each direct import and its dependencies
|
|
205
|
+
for package_name in direct_imports:
|
|
206
|
+
all_packages.append({ 'name': package_name, 'is_direct_import': True })
|
|
207
|
+
# Get all dependencies including sub-dependencies
|
|
208
|
+
dependencies = get_package_dependencies(package_name, processed_packages)
|
|
209
|
+
all_packages.extend([{ 'name': dep, 'is_direct_import': False } for dep in dependencies])
|
|
210
|
+
|
|
211
|
+
# Filter out built-in packages
|
|
212
|
+
return all_packages
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import sys
|
|
3
|
+
from modulefinder import ModuleFinder
|
|
4
|
+
from typing import List
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def trace_project_files(project_dir: str, entry_file: str) -> List[str]:
|
|
8
|
+
"""Trace the project files."""
|
|
9
|
+
|
|
10
|
+
try:
|
|
11
|
+
# Normalize paths
|
|
12
|
+
project_dir = os.path.abspath(project_dir)
|
|
13
|
+
entry_file = os.path.abspath(entry_file)
|
|
14
|
+
|
|
15
|
+
# Create ModuleFinder with default path to find all imports
|
|
16
|
+
finder = ModuleFinder(path=[project_dir])
|
|
17
|
+
|
|
18
|
+
# Run the script to analyze imports
|
|
19
|
+
finder.run_script(entry_file)
|
|
20
|
+
|
|
21
|
+
# Collect all file paths from modules
|
|
22
|
+
project_files = set()
|
|
23
|
+
|
|
24
|
+
for module in finder.modules.values():
|
|
25
|
+
if module is None or not hasattr(module, '__file__') or module.__file__ is None:
|
|
26
|
+
continue
|
|
27
|
+
|
|
28
|
+
# Get the absolute path of the module file
|
|
29
|
+
module_path = os.path.abspath(module.__file__)
|
|
30
|
+
|
|
31
|
+
# Only include files within the project directory
|
|
32
|
+
if module_path.startswith(project_dir) and module_path != entry_file:
|
|
33
|
+
relative_path = os.path.relpath(module_path, project_dir)
|
|
34
|
+
project_files.add(relative_path)
|
|
35
|
+
|
|
36
|
+
# Convert to list and sort for consistent output
|
|
37
|
+
return sorted(list(project_files))
|
|
38
|
+
|
|
39
|
+
except Exception as e:
|
|
40
|
+
print(f"Error in find_project_files: {str(e)}", file=sys.stderr)
|
|
41
|
+
return []
|
|
@@ -5,18 +5,11 @@ import { petStoreService } from './services/pet-store'
|
|
|
5
5
|
export const config: ApiRouteConfig = {
|
|
6
6
|
type: 'api',
|
|
7
7
|
name: 'ApiTrigger',
|
|
8
|
-
description:
|
|
9
|
-
'basic-tutorial api trigger, it uses the petstore public api to create a new pet and emits a topic to proces an order if an item is included.',
|
|
10
|
-
/**
|
|
11
|
-
* The flows this step belongs to, will be available in Workbench
|
|
12
|
-
*/
|
|
8
|
+
description: 'basic-tutorial api trigger',
|
|
13
9
|
flows: ['basic-tutorial'],
|
|
14
10
|
|
|
15
11
|
method: 'POST',
|
|
16
12
|
path: '/basic-tutorial',
|
|
17
|
-
/**
|
|
18
|
-
* Expected request body for type checking and documentation
|
|
19
|
-
*/
|
|
20
13
|
bodySchema: z.object({
|
|
21
14
|
pet: z.object({
|
|
22
15
|
name: z.string(),
|
|
@@ -29,44 +22,22 @@ export const config: ApiRouteConfig = {
|
|
|
29
22
|
})
|
|
30
23
|
.optional(),
|
|
31
24
|
}),
|
|
32
|
-
|
|
33
|
-
/**
|
|
34
|
-
* Expected response body for type checking and documentation
|
|
35
|
-
*/
|
|
36
25
|
responseSchema: {
|
|
37
26
|
200: z.object({
|
|
38
27
|
message: z.string(),
|
|
39
28
|
traceId: z.string(),
|
|
40
29
|
}),
|
|
41
30
|
},
|
|
42
|
-
|
|
43
|
-
/**
|
|
44
|
-
* This API Step emits events to topic `process-food-order`
|
|
45
|
-
*/
|
|
46
31
|
emits: ['process-food-order'],
|
|
47
|
-
|
|
48
|
-
/**
|
|
49
|
-
* We're using virtual subscribes to virtually connect noop step
|
|
50
|
-
* to this step.
|
|
51
|
-
*
|
|
52
|
-
* Noop step is defined in noop.step.ts
|
|
53
|
-
*/
|
|
54
|
-
virtualSubscribes: ['/basic-tutorial'],
|
|
55
32
|
}
|
|
56
33
|
|
|
57
34
|
export const handler: Handlers['ApiTrigger'] = async (req, { logger, emit, traceId }) => {
|
|
58
|
-
/**
|
|
59
|
-
* Avoid usage of console.log, use logger instead
|
|
60
|
-
*/
|
|
61
35
|
logger.info('Step 01 – Processing API Step', { body: req.body })
|
|
62
36
|
|
|
63
37
|
const { pet, foodOrder } = req.body
|
|
64
38
|
|
|
65
39
|
const newPetRecord = await petStoreService.createPet(pet)
|
|
66
40
|
|
|
67
|
-
/**
|
|
68
|
-
* Emit events to the topics to process asynchronously
|
|
69
|
-
*/
|
|
70
41
|
if (foodOrder) {
|
|
71
42
|
await emit({
|
|
72
43
|
topic: 'process-food-order',
|
|
@@ -77,9 +48,6 @@ export const handler: Handlers['ApiTrigger'] = async (req, { logger, emit, trace
|
|
|
77
48
|
})
|
|
78
49
|
}
|
|
79
50
|
|
|
80
|
-
/**
|
|
81
|
-
* Return data back to the client
|
|
82
|
-
*/
|
|
83
51
|
return {
|
|
84
52
|
status: 200,
|
|
85
53
|
body: {
|