motia 0.7.2-beta.134 → 0.7.2-beta.135-685562
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cjs/cloud/build/builders/python/index.d.ts +0 -1
- package/dist/cjs/cloud/build/builders/python/index.js +10 -15
- package/dist/cjs/cloud/build/builders/python/python-data/__tests__/extract-python-data.test.js +4 -5
- package/dist/cjs/cloud/build/builders/python/python-data/__tests__/get-dependencies-from-file.test.js +3 -9
- package/dist/cjs/cloud/build/builders/python/python-data/__tests__/read-requirements.test.js +7 -29
- package/dist/cjs/cloud/build/builders/python/python-data/__tests__/resolve-dep-name.test.js +19 -10
- package/dist/cjs/cloud/build/builders/python/python-data/extract-python-data.d.ts +1 -1
- package/dist/cjs/cloud/build/builders/python/python-data/extract-python-data.js +3 -3
- package/dist/cjs/cloud/build/builders/python/python-data/get-dependencies-from-file.d.ts +1 -2
- package/dist/cjs/cloud/build/builders/python/python-data/get-dependencies-from-file.js +2 -2
- package/dist/cjs/cloud/build/builders/python/python-data/read-requirements.d.ts +8 -10
- package/dist/cjs/cloud/build/builders/python/python-data/read-requirements.js +3 -3
- package/dist/cjs/cloud/build/builders/python/python-data/{resolve-dep-name.d.ts → resolve-dep-names.d.ts} +2 -2
- package/dist/cjs/cloud/build/builders/python/python-data/{resolve-dep-name.js → resolve-dep-names.js} +4 -5
- package/dist/cjs/cloud/build/builders/python/python-data/traverse-tree.d.ts +1 -2
- package/dist/cjs/cloud/build/builders/python/python-data/traverse-tree.js +4 -4
- package/dist/esm/cloud/build/builders/python/index.d.ts +0 -1
- package/dist/esm/cloud/build/builders/python/index.js +10 -15
- package/dist/esm/cloud/build/builders/python/python-data/__tests__/extract-python-data.test.js +4 -5
- package/dist/esm/cloud/build/builders/python/python-data/__tests__/get-dependencies-from-file.test.js +3 -9
- package/dist/esm/cloud/build/builders/python/python-data/__tests__/read-requirements.test.js +7 -29
- package/dist/esm/cloud/build/builders/python/python-data/__tests__/resolve-dep-name.test.js +16 -7
- package/dist/esm/cloud/build/builders/python/python-data/extract-python-data.d.ts +1 -1
- package/dist/esm/cloud/build/builders/python/python-data/extract-python-data.js +3 -3
- package/dist/esm/cloud/build/builders/python/python-data/get-dependencies-from-file.d.ts +1 -2
- package/dist/esm/cloud/build/builders/python/python-data/get-dependencies-from-file.js +2 -2
- package/dist/esm/cloud/build/builders/python/python-data/read-requirements.d.ts +8 -10
- package/dist/esm/cloud/build/builders/python/python-data/read-requirements.js +3 -3
- package/dist/{types/cloud/build/builders/python/python-data/resolve-dep-name.d.ts → esm/cloud/build/builders/python/python-data/resolve-dep-names.d.ts} +2 -2
- package/dist/esm/cloud/build/builders/python/python-data/{resolve-dep-name.js → resolve-dep-names.js} +4 -5
- package/dist/esm/cloud/build/builders/python/python-data/traverse-tree.d.ts +1 -2
- package/dist/esm/cloud/build/builders/python/python-data/traverse-tree.js +4 -4
- package/dist/types/cloud/build/builders/python/index.d.ts +0 -1
- package/dist/types/cloud/build/builders/python/python-data/extract-python-data.d.ts +1 -1
- package/dist/types/cloud/build/builders/python/python-data/get-dependencies-from-file.d.ts +1 -2
- package/dist/types/cloud/build/builders/python/python-data/read-requirements.d.ts +8 -10
- package/dist/{esm/cloud/build/builders/python/python-data/resolve-dep-name.d.ts → types/cloud/build/builders/python/python-data/resolve-dep-names.d.ts} +2 -2
- package/dist/types/cloud/build/builders/python/python-data/traverse-tree.d.ts +1 -2
- package/package.json +4 -4
|
@@ -6,7 +6,6 @@ export declare class PythonBuilder implements StepBuilder {
|
|
|
6
6
|
private readonly listener;
|
|
7
7
|
private packager;
|
|
8
8
|
constructor(builder: Builder, listener: BuildListener);
|
|
9
|
-
private getRequirements;
|
|
10
9
|
buildApiSteps(steps: Step<ApiRouteConfig>[]): Promise<RouterBuildResult>;
|
|
11
10
|
build(step: Step): Promise<void>;
|
|
12
11
|
private generatePackage;
|
|
@@ -12,7 +12,7 @@ const archiver_1 = require("../archiver");
|
|
|
12
12
|
const include_static_files_1 = require("../include-static-files");
|
|
13
13
|
const extract_python_data_1 = require("./python-data/extract-python-data");
|
|
14
14
|
const read_requirements_1 = require("./python-data/read-requirements");
|
|
15
|
-
const
|
|
15
|
+
const resolve_dep_names_1 = require("./python-data/resolve-dep-names");
|
|
16
16
|
const uv_packager_1 = require("./uv-packager");
|
|
17
17
|
class PythonBuilder {
|
|
18
18
|
constructor(builder, listener) {
|
|
@@ -21,17 +21,6 @@ class PythonBuilder {
|
|
|
21
21
|
(0, activate_python_env_1.activatePythonVenv)({ baseDir: this.builder.projectDir });
|
|
22
22
|
this.packager = new uv_packager_1.UvPackager();
|
|
23
23
|
}
|
|
24
|
-
getRequirements() {
|
|
25
|
-
const requirementsFile = path_1.default.join(this.builder.projectDir, 'requirements.txt');
|
|
26
|
-
const depNames = Object.keys((0, read_requirements_1.readRequirements)(requirementsFile, (name) => ({ name, importName: name })));
|
|
27
|
-
const sitePackagesPath = (0, activate_python_env_1.getSitePackagesPath)({ baseDir: this.builder.projectDir });
|
|
28
|
-
const mapper = (0, resolve_dep_name_1.resolveDepNames)(depNames, sitePackagesPath);
|
|
29
|
-
const describer = (name) => {
|
|
30
|
-
const [, to] = mapper.find(([from]) => from === name) ?? [];
|
|
31
|
-
return { name, importName: to ?? name };
|
|
32
|
-
};
|
|
33
|
-
return (0, read_requirements_1.readRequirements)(requirementsFile, describer);
|
|
34
|
-
}
|
|
35
24
|
async buildApiSteps(steps) {
|
|
36
25
|
const zipName = 'router-python.zip';
|
|
37
26
|
const archive = new archiver_1.Archiver(path_1.default.join(constants_1.distDir, zipName));
|
|
@@ -79,8 +68,11 @@ class PythonBuilder {
|
|
|
79
68
|
}
|
|
80
69
|
}
|
|
81
70
|
async generatePackage(bundleDir, entrypointPath, archive, fileContent) {
|
|
82
|
-
const
|
|
83
|
-
const
|
|
71
|
+
const requirementsFile = path_1.default.join(this.builder.projectDir, 'requirements.txt');
|
|
72
|
+
const requirements = (0, read_requirements_1.readRequirements)(requirementsFile);
|
|
73
|
+
const sitePackagesPath = (0, activate_python_env_1.getSitePackagesPath)({ baseDir: this.builder.projectDir });
|
|
74
|
+
const dependenciesMap = (0, resolve_dep_names_1.resolveDepNames)(Object.keys(requirements), sitePackagesPath);
|
|
75
|
+
const { externalDependencies, files } = (0, extract_python_data_1.extractPythonData)(this.builder.projectDir, entrypointPath, dependenciesMap, fileContent);
|
|
84
76
|
// move files
|
|
85
77
|
for (const file of files) {
|
|
86
78
|
fs_1.default.mkdirSync(path_1.default.dirname(path_1.default.join(bundleDir, file)), { recursive: true });
|
|
@@ -94,7 +86,10 @@ class PythonBuilder {
|
|
|
94
86
|
const dependencies = Object.values(externalDependencies);
|
|
95
87
|
if (dependencies.length > 0) {
|
|
96
88
|
// create requirements.txt
|
|
97
|
-
|
|
89
|
+
const requirementsContent = Object.values(externalDependencies)
|
|
90
|
+
.map((dependency) => requirements[dependency])
|
|
91
|
+
.join('\n');
|
|
92
|
+
fs_1.default.writeFileSync(path_1.default.join(bundleDir, 'requirements.txt'), requirementsContent);
|
|
98
93
|
await this.packager.packageDependencies(bundleDir);
|
|
99
94
|
}
|
|
100
95
|
// zip entire folder
|
package/dist/cjs/cloud/build/builders/python/python-data/__tests__/extract-python-data.test.js
CHANGED
|
@@ -8,11 +8,10 @@ const glob_1 = require("glob");
|
|
|
8
8
|
const read_requirements_1 = require("../read-requirements");
|
|
9
9
|
const extract_python_data_1 = require("../extract-python-data");
|
|
10
10
|
const python_errors_1 = require("../python-errors");
|
|
11
|
-
const mockPackageDescriber = (name) => ({ name, importName: name });
|
|
12
11
|
describe('extractPythonData', () => {
|
|
13
12
|
test('extracts python data correctly', () => {
|
|
14
13
|
const rootDir = path_1.default.join(__dirname, './examples/example-1');
|
|
15
|
-
const requirements = (0, read_requirements_1.readRequirements)(path_1.default.join(rootDir, 'requirements.txt')
|
|
14
|
+
const requirements = (0, read_requirements_1.readRequirements)(path_1.default.join(rootDir, 'requirements.txt'));
|
|
16
15
|
const steps = (0, glob_1.globSync)('**/*_step.py', { absolute: false, cwd: path_1.default.join(rootDir, 'steps') });
|
|
17
16
|
for (const file of steps) {
|
|
18
17
|
const result = (0, extract_python_data_1.extractPythonData)(rootDir, `/steps/${file}`, requirements);
|
|
@@ -25,17 +24,17 @@ describe('extractPythonData', () => {
|
|
|
25
24
|
});
|
|
26
25
|
test('extracts python data correctly with invalid dependency', () => {
|
|
27
26
|
const rootDir = path_1.default.join(__dirname, './examples/invalid-dependency');
|
|
28
|
-
const requirements = (0, read_requirements_1.readRequirements)(path_1.default.join(rootDir, 'requirements.txt')
|
|
27
|
+
const requirements = (0, read_requirements_1.readRequirements)(path_1.default.join(rootDir, 'requirements.txt'));
|
|
29
28
|
expect(() => (0, extract_python_data_1.extractPythonData)(rootDir, `/steps/api_step.py`, requirements)).toThrow(python_errors_1.PythonImportNotFoundError);
|
|
30
29
|
});
|
|
31
30
|
test('extracts python data correctly with compilation error', () => {
|
|
32
31
|
const rootDir = path_1.default.join(__dirname, './examples/compilation-error');
|
|
33
|
-
const requirements = (0, read_requirements_1.readRequirements)(path_1.default.join(rootDir, 'requirements.txt')
|
|
32
|
+
const requirements = (0, read_requirements_1.readRequirements)(path_1.default.join(rootDir, 'requirements.txt'));
|
|
34
33
|
expect(() => (0, extract_python_data_1.extractPythonData)(rootDir, `/steps/api_step.py`, requirements)).toThrow(new python_errors_1.PythonError("Compilation error: no viable alternative at input ':' at line 3:10 in /steps/api_step.py", '/steps/api_step.py'));
|
|
35
34
|
});
|
|
36
35
|
test('extracts python data with nested import from requirements', () => {
|
|
37
36
|
const rootDir = path_1.default.join(__dirname, './examples/chessarena');
|
|
38
|
-
const requirements = (0, read_requirements_1.readRequirements)(path_1.default.join(rootDir, 'requirements.txt')
|
|
37
|
+
const requirements = (0, read_requirements_1.readRequirements)(path_1.default.join(rootDir, 'requirements.txt'));
|
|
39
38
|
const result = (0, extract_python_data_1.extractPythonData)(rootDir, `/steps/evaluate_player_move_step.py`, requirements);
|
|
40
39
|
expect(result.externalDependencies).toEqual({ chess: 'chess>=1.0.0', pydantic: 'pydantic>=2.6.1' });
|
|
41
40
|
});
|
|
@@ -11,9 +11,7 @@ import requests
|
|
|
11
11
|
from .local_module import something
|
|
12
12
|
from ..parent_module import other
|
|
13
13
|
`;
|
|
14
|
-
const requirements = {
|
|
15
|
-
requests: 'requests==2.25.1',
|
|
16
|
-
};
|
|
14
|
+
const requirements = { requests: 'requests' };
|
|
17
15
|
const deps = (0, get_dependencies_from_file_1.getDependenciesFromFile)(pythonCode, 'test.py', requirements);
|
|
18
16
|
expect(Array.from(deps.standardLibDependencies)).toEqual(['os', 'sys', 'collections']);
|
|
19
17
|
expect(Array.from(deps.externalDependencies)).toEqual(['requests']);
|
|
@@ -31,9 +29,7 @@ import json
|
|
|
31
29
|
import numpy
|
|
32
30
|
import mypackage.submodule
|
|
33
31
|
`;
|
|
34
|
-
const requirements = {
|
|
35
|
-
numpy: 'numpy==1.21.0',
|
|
36
|
-
};
|
|
32
|
+
const requirements = { numpy: 'numpy' };
|
|
37
33
|
const deps = (0, get_dependencies_from_file_1.getDependenciesFromFile)(pythonCode, 'test.py', requirements);
|
|
38
34
|
expect(Array.from(deps.standardLibDependencies)).toEqual(['json']);
|
|
39
35
|
expect(Array.from(deps.externalDependencies)).toEqual(['numpy']);
|
|
@@ -45,9 +41,7 @@ import json
|
|
|
45
41
|
import numpy as np
|
|
46
42
|
import mypackage.submodule
|
|
47
43
|
`;
|
|
48
|
-
const requirements = {
|
|
49
|
-
numpy: 'numpy==1.21.0',
|
|
50
|
-
};
|
|
44
|
+
const requirements = { numpy: 'numpy' };
|
|
51
45
|
const deps = (0, get_dependencies_from_file_1.getDependenciesFromFile)(pythonCode, 'test.py', requirements);
|
|
52
46
|
expect(Array.from(deps.standardLibDependencies)).toEqual(['json']);
|
|
53
47
|
expect(Array.from(deps.externalDependencies)).toEqual(['numpy']);
|
package/dist/cjs/cloud/build/builders/python/python-data/__tests__/read-requirements.test.js
CHANGED
|
@@ -9,7 +9,6 @@ const path_1 = __importDefault(require("path"));
|
|
|
9
9
|
const os_1 = require("os");
|
|
10
10
|
describe('readRequirements', () => {
|
|
11
11
|
let tempFilePath;
|
|
12
|
-
const mockPackageDescriber = jest.fn((name) => ({ name, importName: name }));
|
|
13
12
|
beforeEach(() => {
|
|
14
13
|
// Create a temporary file path
|
|
15
14
|
tempFilePath = path_1.default.join((0, os_1.tmpdir)(), `requirements-${Date.now()}.txt`);
|
|
@@ -25,7 +24,7 @@ describe('readRequirements', () => {
|
|
|
25
24
|
numpy>=1.20.0
|
|
26
25
|
flask~=2.0`;
|
|
27
26
|
fs_1.default.writeFileSync(tempFilePath, content);
|
|
28
|
-
const requirements = (0, read_requirements_1.readRequirements)(tempFilePath
|
|
27
|
+
const requirements = (0, read_requirements_1.readRequirements)(tempFilePath);
|
|
29
28
|
expect(Object.keys(requirements)).toHaveLength(3);
|
|
30
29
|
expect(requirements.requests).toBe('requests==2.25.1');
|
|
31
30
|
expect(requirements.numpy).toBe('numpy>=1.20.0');
|
|
@@ -40,38 +39,17 @@ numpy>=1.20.0
|
|
|
40
39
|
|
|
41
40
|
flask~=2.0`;
|
|
42
41
|
fs_1.default.writeFileSync(tempFilePath, content);
|
|
43
|
-
const requirements = (0, read_requirements_1.readRequirements)(tempFilePath
|
|
42
|
+
const requirements = (0, read_requirements_1.readRequirements)(tempFilePath);
|
|
44
43
|
expect(Object.keys(requirements)).toHaveLength(3);
|
|
45
44
|
expect(requirements.requests).toBe('requests==2.25.1');
|
|
46
45
|
expect(requirements.numpy).toBe('numpy>=1.20.0');
|
|
47
46
|
expect(requirements.flask).toBe('flask~=2.0');
|
|
48
47
|
});
|
|
49
|
-
test('handles complex package names and version specifiers', () => {
|
|
50
|
-
const content = `scikit-learn==1.0.2
|
|
51
|
-
python-dateutil==1.4
|
|
52
|
-
Django<4.0.0,>=3.2.0`;
|
|
53
|
-
fs_1.default.writeFileSync(tempFilePath, content);
|
|
54
|
-
const describer = jest.fn((name) => {
|
|
55
|
-
// mocking a describer resolution for these packages
|
|
56
|
-
// we're going to implement it separately
|
|
57
|
-
const map = {
|
|
58
|
-
'scikit-learn': 'sklearn',
|
|
59
|
-
'python-dateutil': 'dateutil',
|
|
60
|
-
Django: 'django',
|
|
61
|
-
};
|
|
62
|
-
return map[name] ? { name, importName: map[name] } : { name, importName: name };
|
|
63
|
-
});
|
|
64
|
-
const requirements = (0, read_requirements_1.readRequirements)(tempFilePath, describer);
|
|
65
|
-
expect(Object.keys(requirements)).toEqual(['sklearn', 'dateutil', 'django']);
|
|
66
|
-
expect(requirements['sklearn']).toBe('scikit-learn==1.0.2');
|
|
67
|
-
expect(requirements['dateutil']).toBe('python-dateutil==1.4');
|
|
68
|
-
expect(requirements['django']).toBe('Django<4.0.0,>=3.2.0');
|
|
69
|
-
});
|
|
70
48
|
test('handles requirements with extra dependencies', () => {
|
|
71
49
|
const content = `requests[security]==2.25.1
|
|
72
50
|
flask[async]==2.0.1`;
|
|
73
51
|
fs_1.default.writeFileSync(tempFilePath, content);
|
|
74
|
-
const requirements = (0, read_requirements_1.readRequirements)(tempFilePath
|
|
52
|
+
const requirements = (0, read_requirements_1.readRequirements)(tempFilePath);
|
|
75
53
|
expect(Object.keys(requirements)).toHaveLength(2);
|
|
76
54
|
expect(requirements['requests']).toBe('requests[security]==2.25.1');
|
|
77
55
|
expect(requirements['flask']).toBe('flask[async]==2.0.1');
|
|
@@ -81,7 +59,7 @@ flask[async]==2.0.1`;
|
|
|
81
59
|
|
|
82
60
|
# And empty lines`;
|
|
83
61
|
fs_1.default.writeFileSync(tempFilePath, content);
|
|
84
|
-
const requirements = (0, read_requirements_1.readRequirements)(tempFilePath
|
|
62
|
+
const requirements = (0, read_requirements_1.readRequirements)(tempFilePath);
|
|
85
63
|
expect(Object.keys(requirements)).toHaveLength(0);
|
|
86
64
|
});
|
|
87
65
|
test('handles malformed lines gracefully', () => {
|
|
@@ -90,7 +68,7 @@ flask[async]==2.0.1`;
|
|
|
90
68
|
numpy>=1.20.0
|
|
91
69
|
==invalid-version-only`;
|
|
92
70
|
fs_1.default.writeFileSync(tempFilePath, content);
|
|
93
|
-
const requirements = (0, read_requirements_1.readRequirements)(tempFilePath
|
|
71
|
+
const requirements = (0, read_requirements_1.readRequirements)(tempFilePath);
|
|
94
72
|
expect(Object.keys(requirements)).toHaveLength(2);
|
|
95
73
|
expect(requirements.requests).toBe('requests==2.25.1');
|
|
96
74
|
expect(requirements.numpy).toBe('numpy>=1.20.0');
|
|
@@ -98,7 +76,7 @@ numpy>=1.20.0
|
|
|
98
76
|
test('handles mixed line endings', () => {
|
|
99
77
|
const content = 'requests==2.25.1\r\nnumpy>=1.20.0\nflask~=2.0';
|
|
100
78
|
fs_1.default.writeFileSync(tempFilePath, content);
|
|
101
|
-
const requirements = (0, read_requirements_1.readRequirements)(tempFilePath
|
|
79
|
+
const requirements = (0, read_requirements_1.readRequirements)(tempFilePath);
|
|
102
80
|
expect(Object.keys(requirements)).toHaveLength(3);
|
|
103
81
|
expect(requirements.requests).toBe('requests==2.25.1');
|
|
104
82
|
expect(requirements.numpy).toBe('numpy>=1.20.0');
|
|
@@ -107,7 +85,7 @@ numpy>=1.20.0
|
|
|
107
85
|
test('handles requirements with no versions', () => {
|
|
108
86
|
const content = ['requests', 'numpy', 'flask'].join('\n');
|
|
109
87
|
fs_1.default.writeFileSync(tempFilePath, content);
|
|
110
|
-
const requirements = (0, read_requirements_1.readRequirements)(tempFilePath
|
|
88
|
+
const requirements = (0, read_requirements_1.readRequirements)(tempFilePath);
|
|
111
89
|
expect(Object.keys(requirements)).toHaveLength(3);
|
|
112
90
|
expect(requirements.requests).toBe('requests');
|
|
113
91
|
expect(requirements.numpy).toBe('numpy');
|
|
@@ -4,26 +4,35 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
4
4
|
};
|
|
5
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
const path_1 = __importDefault(require("path"));
|
|
7
|
-
const
|
|
7
|
+
const resolve_dep_names_1 = require("../resolve-dep-names");
|
|
8
8
|
const python_errors_1 = require("../python-errors");
|
|
9
9
|
const sitePackagesDir = path_1.default.join(__dirname, 'site-packages');
|
|
10
10
|
describe('resolveDepName', () => {
|
|
11
11
|
test('resolves dependency names correctly', () => {
|
|
12
12
|
const depNames = ['httpx'];
|
|
13
|
-
const result = (0,
|
|
14
|
-
expect(result).toEqual(
|
|
13
|
+
const result = (0, resolve_dep_names_1.resolveDepNames)(depNames, sitePackagesDir);
|
|
14
|
+
expect(result).toEqual({ httpx: 'httpx' });
|
|
15
15
|
});
|
|
16
16
|
test('resolves multiple dependency names correctly', () => {
|
|
17
17
|
const depNames = ['httpx', 'scikit-learn', 'opencv-python'];
|
|
18
|
-
const result = (0,
|
|
19
|
-
expect(result).toEqual(
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
18
|
+
const result = (0, resolve_dep_names_1.resolveDepNames)(depNames, sitePackagesDir);
|
|
19
|
+
expect(result).toEqual({
|
|
20
|
+
httpx: 'httpx',
|
|
21
|
+
sklearn: 'scikit-learn',
|
|
22
|
+
cv2: 'opencv-python',
|
|
23
|
+
});
|
|
24
|
+
});
|
|
25
|
+
test('resolves pymongo correctly', () => {
|
|
26
|
+
const depNames = ['pymongo'];
|
|
27
|
+
const result = (0, resolve_dep_names_1.resolveDepNames)(depNames, sitePackagesDir);
|
|
28
|
+
expect(result).toEqual({
|
|
29
|
+
pymongo: 'pymongo',
|
|
30
|
+
bson: 'pymongo',
|
|
31
|
+
gridfs: 'pymongo',
|
|
32
|
+
});
|
|
24
33
|
});
|
|
25
34
|
test('should throw an error if the dependency is not found', () => {
|
|
26
35
|
const depNames = ['httpx', 'scikit-learn', 'opencv-python', 'pydantic'];
|
|
27
|
-
expect(() => (0,
|
|
36
|
+
expect(() => (0, resolve_dep_names_1.resolveDepNames)(depNames, sitePackagesDir)).toThrow(new python_errors_1.PythonError('Could not find dependency name in site-packages: pydantic', 'pydantic'));
|
|
28
37
|
});
|
|
29
38
|
});
|
|
@@ -4,4 +4,4 @@ export type PythonResult = {
|
|
|
4
4
|
externalDependencies: Requirements;
|
|
5
5
|
files: string[];
|
|
6
6
|
};
|
|
7
|
-
export declare const extractPythonData: (rootDir: string, filePath: string,
|
|
7
|
+
export declare const extractPythonData: (rootDir: string, filePath: string, dependenciesMap: Record<string, string>, fileContent?: string) => PythonResult;
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.extractPythonData = void 0;
|
|
4
4
|
const traverse_tree_1 = require("./traverse-tree");
|
|
5
|
-
const extractPythonData = (rootDir, filePath,
|
|
5
|
+
const extractPythonData = (rootDir, filePath, dependenciesMap,
|
|
6
6
|
// optional
|
|
7
7
|
fileContent) => {
|
|
8
8
|
const result = {
|
|
@@ -10,10 +10,10 @@ fileContent) => {
|
|
|
10
10
|
externalDependencies: new Set(),
|
|
11
11
|
files: new Set(),
|
|
12
12
|
};
|
|
13
|
-
(0, traverse_tree_1.traverseTree)(rootDir, filePath, result,
|
|
13
|
+
(0, traverse_tree_1.traverseTree)(rootDir, filePath, result, dependenciesMap, fileContent);
|
|
14
14
|
const resultDependencies = {};
|
|
15
15
|
result.externalDependencies.forEach((dependency) => {
|
|
16
|
-
resultDependencies[dependency] =
|
|
16
|
+
resultDependencies[dependency] = dependenciesMap[dependency];
|
|
17
17
|
});
|
|
18
18
|
return {
|
|
19
19
|
standardLibDependencies: Array.from(result.standardLibDependencies),
|
|
@@ -1,7 +1,6 @@
|
|
|
1
|
-
import { Requirements } from './read-requirements';
|
|
2
1
|
export type Dependencies = {
|
|
3
2
|
standardLibDependencies: Set<string>;
|
|
4
3
|
externalDependencies: Set<string>;
|
|
5
4
|
projectDependencies: Set<string>;
|
|
6
5
|
};
|
|
7
|
-
export declare const getDependenciesFromFile: (content: string, path: string,
|
|
6
|
+
export declare const getDependenciesFromFile: (content: string, path: string, externalDependenciesMap: Record<string, string>) => Dependencies;
|
|
@@ -23,7 +23,7 @@ function parse(source, sourceName) {
|
|
|
23
23
|
parser.addErrorListener(listener);
|
|
24
24
|
return parser.file_input();
|
|
25
25
|
}
|
|
26
|
-
const getDependenciesFromFile = (content, path,
|
|
26
|
+
const getDependenciesFromFile = (content, path, externalDependenciesMap) => {
|
|
27
27
|
const result = parse(content + '\n', path);
|
|
28
28
|
const modulesSet = new Set();
|
|
29
29
|
(0, python_ast_1.createVisitor)({
|
|
@@ -60,7 +60,7 @@ const getDependenciesFromFile = (content, path, requirements) => {
|
|
|
60
60
|
else if (constants_1.STANDARD_LIB_MODULES.has(module)) {
|
|
61
61
|
dependencies.standardLibDependencies.add(module);
|
|
62
62
|
}
|
|
63
|
-
else if (
|
|
63
|
+
else if (externalDependenciesMap[module] || externalDependenciesMap[moduleName]) {
|
|
64
64
|
dependencies.externalDependencies.add(module);
|
|
65
65
|
}
|
|
66
66
|
else {
|
|
@@ -1,15 +1,13 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
type PackageDescriber = (packageName: string) => PackageInfo;
|
|
1
|
+
/**
|
|
2
|
+
* { libraryName: lineFromRequirementsFile }
|
|
3
|
+
*
|
|
4
|
+
* Example:
|
|
5
|
+
* { scikit-learn: 'scikit-learn==1.0.2' }
|
|
6
|
+
*/
|
|
7
|
+
export type Requirements = Record<string, string>;
|
|
9
8
|
/**
|
|
10
9
|
* Read the requirements.txt file and return a set of dependencies
|
|
11
10
|
* @param filePath
|
|
12
11
|
* @returns set of all dependencies names
|
|
13
12
|
*/
|
|
14
|
-
export declare const readRequirements: (filePath: string
|
|
15
|
-
export {};
|
|
13
|
+
export declare const readRequirements: (filePath: string) => Requirements;
|
|
@@ -10,7 +10,7 @@ const fs_1 = __importDefault(require("fs"));
|
|
|
10
10
|
* @param filePath
|
|
11
11
|
* @returns set of all dependencies names
|
|
12
12
|
*/
|
|
13
|
-
const readRequirements = (filePath
|
|
13
|
+
const readRequirements = (filePath) => {
|
|
14
14
|
const content = fs_1.default.readFileSync(filePath, 'utf8');
|
|
15
15
|
const lines = content.split('\n');
|
|
16
16
|
const requirements = {};
|
|
@@ -23,8 +23,8 @@ const readRequirements = (filePath, describer) => {
|
|
|
23
23
|
// Extract package name (everything before version specifiers)
|
|
24
24
|
const packageMatch = trimmedLine.match(/^([a-zA-Z0-9_-]+)/);
|
|
25
25
|
if (packageMatch) {
|
|
26
|
-
const
|
|
27
|
-
requirements[
|
|
26
|
+
const packageName = packageMatch[1];
|
|
27
|
+
requirements[packageName] = trimmedLine;
|
|
28
28
|
}
|
|
29
29
|
}
|
|
30
30
|
return requirements;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Resolve the name of the import from the name of the package
|
|
3
3
|
* @param depName the package name
|
|
4
|
-
* @returns the import name
|
|
4
|
+
* @returns a map of the import name to the package name
|
|
5
5
|
*/
|
|
6
|
-
export declare const resolveDepNames: (depNames: string[], sitePackagesDir: string) =>
|
|
6
|
+
export declare const resolveDepNames: (depNames: string[], sitePackagesDir: string) => Record<string, string>;
|
|
@@ -10,13 +10,13 @@ const python_errors_1 = require("./python-errors");
|
|
|
10
10
|
/**
|
|
11
11
|
* Resolve the name of the import from the name of the package
|
|
12
12
|
* @param depName the package name
|
|
13
|
-
* @returns the import name
|
|
13
|
+
* @returns a map of the import name to the package name
|
|
14
14
|
*/
|
|
15
15
|
const resolveDepNames = (depNames, sitePackagesDir) => {
|
|
16
16
|
const folders = fs_1.default
|
|
17
17
|
.readdirSync(sitePackagesDir)
|
|
18
18
|
.filter((folder) => fs_1.default.statSync(path_1.default.join(sitePackagesDir, folder)).isDirectory());
|
|
19
|
-
const result =
|
|
19
|
+
const result = {};
|
|
20
20
|
for (const depName of depNames) {
|
|
21
21
|
const regex = new RegExp(`^${depName.replace(/-/g, '[-_]')}-.*\\.dist-info$`);
|
|
22
22
|
const folder = folders.find((folder) => regex.test(folder));
|
|
@@ -25,7 +25,7 @@ const resolveDepNames = (depNames, sitePackagesDir) => {
|
|
|
25
25
|
const recordFile = fs_1.default.existsSync(path_1.default.join(sitePackagesDir, folder, 'RECORD'));
|
|
26
26
|
if (topLevelFile) {
|
|
27
27
|
const topLevel = fs_1.default.readFileSync(path_1.default.join(sitePackagesDir, folder, 'top_level.txt'), 'utf8').trim();
|
|
28
|
-
result
|
|
28
|
+
result[topLevel] = depName;
|
|
29
29
|
}
|
|
30
30
|
else if (recordFile) {
|
|
31
31
|
const record = fs_1.default.readFileSync(path_1.default.join(sitePackagesDir, folder, 'RECORD'), 'utf8').trim();
|
|
@@ -35,8 +35,7 @@ const resolveDepNames = (depNames, sitePackagesDir) => {
|
|
|
35
35
|
// RECORD format: path/__init__.py,sha256=hash,size
|
|
36
36
|
const match = /^([^/]+)\/__init__\.py,/.exec(line);
|
|
37
37
|
if (match) {
|
|
38
|
-
result
|
|
39
|
-
break;
|
|
38
|
+
result[match[1]] = depName;
|
|
40
39
|
}
|
|
41
40
|
}
|
|
42
41
|
}
|
|
@@ -1,7 +1,6 @@
|
|
|
1
|
-
import { Requirements } from './read-requirements';
|
|
2
1
|
export type TraverseTreeResult = {
|
|
3
2
|
standardLibDependencies: Set<string>;
|
|
4
3
|
externalDependencies: Set<string>;
|
|
5
4
|
files: Set<string>;
|
|
6
5
|
};
|
|
7
|
-
export declare const traverseTree: (rootDir: string, filePath: string, result: TraverseTreeResult,
|
|
6
|
+
export declare const traverseTree: (rootDir: string, filePath: string, result: TraverseTreeResult, dependenciesMap: Record<string, string>, fileContent?: string) => void;
|
|
@@ -9,7 +9,7 @@ const path_1 = __importDefault(require("path"));
|
|
|
9
9
|
const convert_import_path_1 = require("./convert-import-path");
|
|
10
10
|
const get_dependencies_from_file_1 = require("./get-dependencies-from-file");
|
|
11
11
|
const python_errors_1 = require("./python-errors");
|
|
12
|
-
const traverseTree = (rootDir, filePath, result,
|
|
12
|
+
const traverseTree = (rootDir, filePath, result, dependenciesMap,
|
|
13
13
|
// optional
|
|
14
14
|
fileContent) => {
|
|
15
15
|
const fileAbsolutePath = path_1.default.join(rootDir, filePath);
|
|
@@ -21,7 +21,7 @@ fileContent) => {
|
|
|
21
21
|
result.files.add(initPath.replace(rootDir, ''));
|
|
22
22
|
}
|
|
23
23
|
const content = fileContent || fs_1.default.readFileSync(fileAbsolutePath, 'utf8');
|
|
24
|
-
const dependencies = (0, get_dependencies_from_file_1.getDependenciesFromFile)(content, filePath,
|
|
24
|
+
const dependencies = (0, get_dependencies_from_file_1.getDependenciesFromFile)(content, filePath, dependenciesMap);
|
|
25
25
|
result.files.add(filePath);
|
|
26
26
|
dependencies.externalDependencies.forEach((dependency) => {
|
|
27
27
|
result.externalDependencies.add(dependency);
|
|
@@ -36,7 +36,7 @@ fileContent) => {
|
|
|
36
36
|
const dependencyPath = dependencyFilePath.replace(rootDir, '');
|
|
37
37
|
if (!result.files.has(dependencyPath)) {
|
|
38
38
|
try {
|
|
39
|
-
(0, exports.traverseTree)(rootDir, dependencyPath, result,
|
|
39
|
+
(0, exports.traverseTree)(rootDir, dependencyPath, result, dependenciesMap);
|
|
40
40
|
}
|
|
41
41
|
catch (error) {
|
|
42
42
|
if (error instanceof python_errors_1.PythonFileNotFoundError) {
|
|
@@ -44,7 +44,7 @@ fileContent) => {
|
|
|
44
44
|
// try root folder
|
|
45
45
|
try {
|
|
46
46
|
const rootDependencyFilePath = path_1.default.resolve(rootDir, `${pythonPath}.py`).replace(rootDir, '');
|
|
47
|
-
return (0, exports.traverseTree)(rootDir, rootDependencyFilePath, result,
|
|
47
|
+
return (0, exports.traverseTree)(rootDir, rootDependencyFilePath, result, dependenciesMap);
|
|
48
48
|
}
|
|
49
49
|
catch (_error) {
|
|
50
50
|
// let it throw
|
|
@@ -6,7 +6,6 @@ export declare class PythonBuilder implements StepBuilder {
|
|
|
6
6
|
private readonly listener;
|
|
7
7
|
private packager;
|
|
8
8
|
constructor(builder: Builder, listener: BuildListener);
|
|
9
|
-
private getRequirements;
|
|
10
9
|
buildApiSteps(steps: Step<ApiRouteConfig>[]): Promise<RouterBuildResult>;
|
|
11
10
|
build(step: Step): Promise<void>;
|
|
12
11
|
private generatePackage;
|
|
@@ -6,7 +6,7 @@ import { Archiver } from '../archiver';
|
|
|
6
6
|
import { includeStaticFiles } from '../include-static-files';
|
|
7
7
|
import { extractPythonData } from './python-data/extract-python-data';
|
|
8
8
|
import { readRequirements } from './python-data/read-requirements';
|
|
9
|
-
import { resolveDepNames } from './python-data/resolve-dep-
|
|
9
|
+
import { resolveDepNames } from './python-data/resolve-dep-names';
|
|
10
10
|
import { UvPackager } from './uv-packager';
|
|
11
11
|
export class PythonBuilder {
|
|
12
12
|
constructor(builder, listener) {
|
|
@@ -15,17 +15,6 @@ export class PythonBuilder {
|
|
|
15
15
|
activatePythonVenv({ baseDir: this.builder.projectDir });
|
|
16
16
|
this.packager = new UvPackager();
|
|
17
17
|
}
|
|
18
|
-
getRequirements() {
|
|
19
|
-
const requirementsFile = path.join(this.builder.projectDir, 'requirements.txt');
|
|
20
|
-
const depNames = Object.keys(readRequirements(requirementsFile, (name) => ({ name, importName: name })));
|
|
21
|
-
const sitePackagesPath = getSitePackagesPath({ baseDir: this.builder.projectDir });
|
|
22
|
-
const mapper = resolveDepNames(depNames, sitePackagesPath);
|
|
23
|
-
const describer = (name) => {
|
|
24
|
-
const [, to] = mapper.find(([from]) => from === name) ?? [];
|
|
25
|
-
return { name, importName: to ?? name };
|
|
26
|
-
};
|
|
27
|
-
return readRequirements(requirementsFile, describer);
|
|
28
|
-
}
|
|
29
18
|
async buildApiSteps(steps) {
|
|
30
19
|
const zipName = 'router-python.zip';
|
|
31
20
|
const archive = new Archiver(path.join(distDir, zipName));
|
|
@@ -73,8 +62,11 @@ export class PythonBuilder {
|
|
|
73
62
|
}
|
|
74
63
|
}
|
|
75
64
|
async generatePackage(bundleDir, entrypointPath, archive, fileContent) {
|
|
76
|
-
const
|
|
77
|
-
const
|
|
65
|
+
const requirementsFile = path.join(this.builder.projectDir, 'requirements.txt');
|
|
66
|
+
const requirements = readRequirements(requirementsFile);
|
|
67
|
+
const sitePackagesPath = getSitePackagesPath({ baseDir: this.builder.projectDir });
|
|
68
|
+
const dependenciesMap = resolveDepNames(Object.keys(requirements), sitePackagesPath);
|
|
69
|
+
const { externalDependencies, files } = extractPythonData(this.builder.projectDir, entrypointPath, dependenciesMap, fileContent);
|
|
78
70
|
// move files
|
|
79
71
|
for (const file of files) {
|
|
80
72
|
fs.mkdirSync(path.dirname(path.join(bundleDir, file)), { recursive: true });
|
|
@@ -88,7 +80,10 @@ export class PythonBuilder {
|
|
|
88
80
|
const dependencies = Object.values(externalDependencies);
|
|
89
81
|
if (dependencies.length > 0) {
|
|
90
82
|
// create requirements.txt
|
|
91
|
-
|
|
83
|
+
const requirementsContent = Object.values(externalDependencies)
|
|
84
|
+
.map((dependency) => requirements[dependency])
|
|
85
|
+
.join('\n');
|
|
86
|
+
fs.writeFileSync(path.join(bundleDir, 'requirements.txt'), requirementsContent);
|
|
92
87
|
await this.packager.packageDependencies(bundleDir);
|
|
93
88
|
}
|
|
94
89
|
// zip entire folder
|
package/dist/esm/cloud/build/builders/python/python-data/__tests__/extract-python-data.test.js
CHANGED
|
@@ -3,11 +3,10 @@ import { globSync } from 'glob';
|
|
|
3
3
|
import { readRequirements } from '../read-requirements';
|
|
4
4
|
import { extractPythonData } from '../extract-python-data';
|
|
5
5
|
import { PythonError, PythonImportNotFoundError } from '../python-errors';
|
|
6
|
-
const mockPackageDescriber = (name) => ({ name, importName: name });
|
|
7
6
|
describe('extractPythonData', () => {
|
|
8
7
|
test('extracts python data correctly', () => {
|
|
9
8
|
const rootDir = path.join(__dirname, './examples/example-1');
|
|
10
|
-
const requirements = readRequirements(path.join(rootDir, 'requirements.txt')
|
|
9
|
+
const requirements = readRequirements(path.join(rootDir, 'requirements.txt'));
|
|
11
10
|
const steps = globSync('**/*_step.py', { absolute: false, cwd: path.join(rootDir, 'steps') });
|
|
12
11
|
for (const file of steps) {
|
|
13
12
|
const result = extractPythonData(rootDir, `/steps/${file}`, requirements);
|
|
@@ -20,17 +19,17 @@ describe('extractPythonData', () => {
|
|
|
20
19
|
});
|
|
21
20
|
test('extracts python data correctly with invalid dependency', () => {
|
|
22
21
|
const rootDir = path.join(__dirname, './examples/invalid-dependency');
|
|
23
|
-
const requirements = readRequirements(path.join(rootDir, 'requirements.txt')
|
|
22
|
+
const requirements = readRequirements(path.join(rootDir, 'requirements.txt'));
|
|
24
23
|
expect(() => extractPythonData(rootDir, `/steps/api_step.py`, requirements)).toThrow(PythonImportNotFoundError);
|
|
25
24
|
});
|
|
26
25
|
test('extracts python data correctly with compilation error', () => {
|
|
27
26
|
const rootDir = path.join(__dirname, './examples/compilation-error');
|
|
28
|
-
const requirements = readRequirements(path.join(rootDir, 'requirements.txt')
|
|
27
|
+
const requirements = readRequirements(path.join(rootDir, 'requirements.txt'));
|
|
29
28
|
expect(() => extractPythonData(rootDir, `/steps/api_step.py`, requirements)).toThrow(new PythonError("Compilation error: no viable alternative at input ':' at line 3:10 in /steps/api_step.py", '/steps/api_step.py'));
|
|
30
29
|
});
|
|
31
30
|
test('extracts python data with nested import from requirements', () => {
|
|
32
31
|
const rootDir = path.join(__dirname, './examples/chessarena');
|
|
33
|
-
const requirements = readRequirements(path.join(rootDir, 'requirements.txt')
|
|
32
|
+
const requirements = readRequirements(path.join(rootDir, 'requirements.txt'));
|
|
34
33
|
const result = extractPythonData(rootDir, `/steps/evaluate_player_move_step.py`, requirements);
|
|
35
34
|
expect(result.externalDependencies).toEqual({ chess: 'chess>=1.0.0', pydantic: 'pydantic>=2.6.1' });
|
|
36
35
|
});
|
|
@@ -9,9 +9,7 @@ import requests
|
|
|
9
9
|
from .local_module import something
|
|
10
10
|
from ..parent_module import other
|
|
11
11
|
`;
|
|
12
|
-
const requirements = {
|
|
13
|
-
requests: 'requests==2.25.1',
|
|
14
|
-
};
|
|
12
|
+
const requirements = { requests: 'requests' };
|
|
15
13
|
const deps = getDependenciesFromFile(pythonCode, 'test.py', requirements);
|
|
16
14
|
expect(Array.from(deps.standardLibDependencies)).toEqual(['os', 'sys', 'collections']);
|
|
17
15
|
expect(Array.from(deps.externalDependencies)).toEqual(['requests']);
|
|
@@ -29,9 +27,7 @@ import json
|
|
|
29
27
|
import numpy
|
|
30
28
|
import mypackage.submodule
|
|
31
29
|
`;
|
|
32
|
-
const requirements = {
|
|
33
|
-
numpy: 'numpy==1.21.0',
|
|
34
|
-
};
|
|
30
|
+
const requirements = { numpy: 'numpy' };
|
|
35
31
|
const deps = getDependenciesFromFile(pythonCode, 'test.py', requirements);
|
|
36
32
|
expect(Array.from(deps.standardLibDependencies)).toEqual(['json']);
|
|
37
33
|
expect(Array.from(deps.externalDependencies)).toEqual(['numpy']);
|
|
@@ -43,9 +39,7 @@ import json
|
|
|
43
39
|
import numpy as np
|
|
44
40
|
import mypackage.submodule
|
|
45
41
|
`;
|
|
46
|
-
const requirements = {
|
|
47
|
-
numpy: 'numpy==1.21.0',
|
|
48
|
-
};
|
|
42
|
+
const requirements = { numpy: 'numpy' };
|
|
49
43
|
const deps = getDependenciesFromFile(pythonCode, 'test.py', requirements);
|
|
50
44
|
expect(Array.from(deps.standardLibDependencies)).toEqual(['json']);
|
|
51
45
|
expect(Array.from(deps.externalDependencies)).toEqual(['numpy']);
|
package/dist/esm/cloud/build/builders/python/python-data/__tests__/read-requirements.test.js
CHANGED
|
@@ -4,7 +4,6 @@ import path from 'path';
|
|
|
4
4
|
import { tmpdir } from 'os';
|
|
5
5
|
describe('readRequirements', () => {
|
|
6
6
|
let tempFilePath;
|
|
7
|
-
const mockPackageDescriber = jest.fn((name) => ({ name, importName: name }));
|
|
8
7
|
beforeEach(() => {
|
|
9
8
|
// Create a temporary file path
|
|
10
9
|
tempFilePath = path.join(tmpdir(), `requirements-${Date.now()}.txt`);
|
|
@@ -20,7 +19,7 @@ describe('readRequirements', () => {
|
|
|
20
19
|
numpy>=1.20.0
|
|
21
20
|
flask~=2.0`;
|
|
22
21
|
fs.writeFileSync(tempFilePath, content);
|
|
23
|
-
const requirements = readRequirements(tempFilePath
|
|
22
|
+
const requirements = readRequirements(tempFilePath);
|
|
24
23
|
expect(Object.keys(requirements)).toHaveLength(3);
|
|
25
24
|
expect(requirements.requests).toBe('requests==2.25.1');
|
|
26
25
|
expect(requirements.numpy).toBe('numpy>=1.20.0');
|
|
@@ -35,38 +34,17 @@ numpy>=1.20.0
|
|
|
35
34
|
|
|
36
35
|
flask~=2.0`;
|
|
37
36
|
fs.writeFileSync(tempFilePath, content);
|
|
38
|
-
const requirements = readRequirements(tempFilePath
|
|
37
|
+
const requirements = readRequirements(tempFilePath);
|
|
39
38
|
expect(Object.keys(requirements)).toHaveLength(3);
|
|
40
39
|
expect(requirements.requests).toBe('requests==2.25.1');
|
|
41
40
|
expect(requirements.numpy).toBe('numpy>=1.20.0');
|
|
42
41
|
expect(requirements.flask).toBe('flask~=2.0');
|
|
43
42
|
});
|
|
44
|
-
test('handles complex package names and version specifiers', () => {
|
|
45
|
-
const content = `scikit-learn==1.0.2
|
|
46
|
-
python-dateutil==1.4
|
|
47
|
-
Django<4.0.0,>=3.2.0`;
|
|
48
|
-
fs.writeFileSync(tempFilePath, content);
|
|
49
|
-
const describer = jest.fn((name) => {
|
|
50
|
-
// mocking a describer resolution for these packages
|
|
51
|
-
// we're going to implement it separately
|
|
52
|
-
const map = {
|
|
53
|
-
'scikit-learn': 'sklearn',
|
|
54
|
-
'python-dateutil': 'dateutil',
|
|
55
|
-
Django: 'django',
|
|
56
|
-
};
|
|
57
|
-
return map[name] ? { name, importName: map[name] } : { name, importName: name };
|
|
58
|
-
});
|
|
59
|
-
const requirements = readRequirements(tempFilePath, describer);
|
|
60
|
-
expect(Object.keys(requirements)).toEqual(['sklearn', 'dateutil', 'django']);
|
|
61
|
-
expect(requirements['sklearn']).toBe('scikit-learn==1.0.2');
|
|
62
|
-
expect(requirements['dateutil']).toBe('python-dateutil==1.4');
|
|
63
|
-
expect(requirements['django']).toBe('Django<4.0.0,>=3.2.0');
|
|
64
|
-
});
|
|
65
43
|
test('handles requirements with extra dependencies', () => {
|
|
66
44
|
const content = `requests[security]==2.25.1
|
|
67
45
|
flask[async]==2.0.1`;
|
|
68
46
|
fs.writeFileSync(tempFilePath, content);
|
|
69
|
-
const requirements = readRequirements(tempFilePath
|
|
47
|
+
const requirements = readRequirements(tempFilePath);
|
|
70
48
|
expect(Object.keys(requirements)).toHaveLength(2);
|
|
71
49
|
expect(requirements['requests']).toBe('requests[security]==2.25.1');
|
|
72
50
|
expect(requirements['flask']).toBe('flask[async]==2.0.1');
|
|
@@ -76,7 +54,7 @@ flask[async]==2.0.1`;
|
|
|
76
54
|
|
|
77
55
|
# And empty lines`;
|
|
78
56
|
fs.writeFileSync(tempFilePath, content);
|
|
79
|
-
const requirements = readRequirements(tempFilePath
|
|
57
|
+
const requirements = readRequirements(tempFilePath);
|
|
80
58
|
expect(Object.keys(requirements)).toHaveLength(0);
|
|
81
59
|
});
|
|
82
60
|
test('handles malformed lines gracefully', () => {
|
|
@@ -85,7 +63,7 @@ flask[async]==2.0.1`;
|
|
|
85
63
|
numpy>=1.20.0
|
|
86
64
|
==invalid-version-only`;
|
|
87
65
|
fs.writeFileSync(tempFilePath, content);
|
|
88
|
-
const requirements = readRequirements(tempFilePath
|
|
66
|
+
const requirements = readRequirements(tempFilePath);
|
|
89
67
|
expect(Object.keys(requirements)).toHaveLength(2);
|
|
90
68
|
expect(requirements.requests).toBe('requests==2.25.1');
|
|
91
69
|
expect(requirements.numpy).toBe('numpy>=1.20.0');
|
|
@@ -93,7 +71,7 @@ numpy>=1.20.0
|
|
|
93
71
|
test('handles mixed line endings', () => {
|
|
94
72
|
const content = 'requests==2.25.1\r\nnumpy>=1.20.0\nflask~=2.0';
|
|
95
73
|
fs.writeFileSync(tempFilePath, content);
|
|
96
|
-
const requirements = readRequirements(tempFilePath
|
|
74
|
+
const requirements = readRequirements(tempFilePath);
|
|
97
75
|
expect(Object.keys(requirements)).toHaveLength(3);
|
|
98
76
|
expect(requirements.requests).toBe('requests==2.25.1');
|
|
99
77
|
expect(requirements.numpy).toBe('numpy>=1.20.0');
|
|
@@ -102,7 +80,7 @@ numpy>=1.20.0
|
|
|
102
80
|
test('handles requirements with no versions', () => {
|
|
103
81
|
const content = ['requests', 'numpy', 'flask'].join('\n');
|
|
104
82
|
fs.writeFileSync(tempFilePath, content);
|
|
105
|
-
const requirements = readRequirements(tempFilePath
|
|
83
|
+
const requirements = readRequirements(tempFilePath);
|
|
106
84
|
expect(Object.keys(requirements)).toHaveLength(3);
|
|
107
85
|
expect(requirements.requests).toBe('requests');
|
|
108
86
|
expect(requirements.numpy).toBe('numpy');
|
|
@@ -1,21 +1,30 @@
|
|
|
1
1
|
import path from 'path';
|
|
2
|
-
import { resolveDepNames } from '../resolve-dep-
|
|
2
|
+
import { resolveDepNames } from '../resolve-dep-names';
|
|
3
3
|
import { PythonError } from '../python-errors';
|
|
4
4
|
const sitePackagesDir = path.join(__dirname, 'site-packages');
|
|
5
5
|
describe('resolveDepName', () => {
|
|
6
6
|
test('resolves dependency names correctly', () => {
|
|
7
7
|
const depNames = ['httpx'];
|
|
8
8
|
const result = resolveDepNames(depNames, sitePackagesDir);
|
|
9
|
-
expect(result).toEqual(
|
|
9
|
+
expect(result).toEqual({ httpx: 'httpx' });
|
|
10
10
|
});
|
|
11
11
|
test('resolves multiple dependency names correctly', () => {
|
|
12
12
|
const depNames = ['httpx', 'scikit-learn', 'opencv-python'];
|
|
13
13
|
const result = resolveDepNames(depNames, sitePackagesDir);
|
|
14
|
-
expect(result).toEqual(
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
14
|
+
expect(result).toEqual({
|
|
15
|
+
httpx: 'httpx',
|
|
16
|
+
sklearn: 'scikit-learn',
|
|
17
|
+
cv2: 'opencv-python',
|
|
18
|
+
});
|
|
19
|
+
});
|
|
20
|
+
test('resolves pymongo correctly', () => {
|
|
21
|
+
const depNames = ['pymongo'];
|
|
22
|
+
const result = resolveDepNames(depNames, sitePackagesDir);
|
|
23
|
+
expect(result).toEqual({
|
|
24
|
+
pymongo: 'pymongo',
|
|
25
|
+
bson: 'pymongo',
|
|
26
|
+
gridfs: 'pymongo',
|
|
27
|
+
});
|
|
19
28
|
});
|
|
20
29
|
test('should throw an error if the dependency is not found', () => {
|
|
21
30
|
const depNames = ['httpx', 'scikit-learn', 'opencv-python', 'pydantic'];
|
|
@@ -4,4 +4,4 @@ export type PythonResult = {
|
|
|
4
4
|
externalDependencies: Requirements;
|
|
5
5
|
files: string[];
|
|
6
6
|
};
|
|
7
|
-
export declare const extractPythonData: (rootDir: string, filePath: string,
|
|
7
|
+
export declare const extractPythonData: (rootDir: string, filePath: string, dependenciesMap: Record<string, string>, fileContent?: string) => PythonResult;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { traverseTree } from './traverse-tree';
|
|
2
|
-
export const extractPythonData = (rootDir, filePath,
|
|
2
|
+
export const extractPythonData = (rootDir, filePath, dependenciesMap,
|
|
3
3
|
// optional
|
|
4
4
|
fileContent) => {
|
|
5
5
|
const result = {
|
|
@@ -7,10 +7,10 @@ fileContent) => {
|
|
|
7
7
|
externalDependencies: new Set(),
|
|
8
8
|
files: new Set(),
|
|
9
9
|
};
|
|
10
|
-
traverseTree(rootDir, filePath, result,
|
|
10
|
+
traverseTree(rootDir, filePath, result, dependenciesMap, fileContent);
|
|
11
11
|
const resultDependencies = {};
|
|
12
12
|
result.externalDependencies.forEach((dependency) => {
|
|
13
|
-
resultDependencies[dependency] =
|
|
13
|
+
resultDependencies[dependency] = dependenciesMap[dependency];
|
|
14
14
|
});
|
|
15
15
|
return {
|
|
16
16
|
standardLibDependencies: Array.from(result.standardLibDependencies),
|
|
@@ -1,7 +1,6 @@
|
|
|
1
|
-
import { Requirements } from './read-requirements';
|
|
2
1
|
export type Dependencies = {
|
|
3
2
|
standardLibDependencies: Set<string>;
|
|
4
3
|
externalDependencies: Set<string>;
|
|
5
4
|
projectDependencies: Set<string>;
|
|
6
5
|
};
|
|
7
|
-
export declare const getDependenciesFromFile: (content: string, path: string,
|
|
6
|
+
export declare const getDependenciesFromFile: (content: string, path: string, externalDependenciesMap: Record<string, string>) => Dependencies;
|
|
@@ -20,7 +20,7 @@ function parse(source, sourceName) {
|
|
|
20
20
|
parser.addErrorListener(listener);
|
|
21
21
|
return parser.file_input();
|
|
22
22
|
}
|
|
23
|
-
export const getDependenciesFromFile = (content, path,
|
|
23
|
+
export const getDependenciesFromFile = (content, path, externalDependenciesMap) => {
|
|
24
24
|
const result = parse(content + '\n', path);
|
|
25
25
|
const modulesSet = new Set();
|
|
26
26
|
createVisitor({
|
|
@@ -57,7 +57,7 @@ export const getDependenciesFromFile = (content, path, requirements) => {
|
|
|
57
57
|
else if (STANDARD_LIB_MODULES.has(module)) {
|
|
58
58
|
dependencies.standardLibDependencies.add(module);
|
|
59
59
|
}
|
|
60
|
-
else if (
|
|
60
|
+
else if (externalDependenciesMap[module] || externalDependenciesMap[moduleName]) {
|
|
61
61
|
dependencies.externalDependencies.add(module);
|
|
62
62
|
}
|
|
63
63
|
else {
|
|
@@ -1,15 +1,13 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
type PackageDescriber = (packageName: string) => PackageInfo;
|
|
1
|
+
/**
|
|
2
|
+
* { libraryName: lineFromRequirementsFile }
|
|
3
|
+
*
|
|
4
|
+
* Example:
|
|
5
|
+
* { scikit-learn: 'scikit-learn==1.0.2' }
|
|
6
|
+
*/
|
|
7
|
+
export type Requirements = Record<string, string>;
|
|
9
8
|
/**
|
|
10
9
|
* Read the requirements.txt file and return a set of dependencies
|
|
11
10
|
* @param filePath
|
|
12
11
|
* @returns set of all dependencies names
|
|
13
12
|
*/
|
|
14
|
-
export declare const readRequirements: (filePath: string
|
|
15
|
-
export {};
|
|
13
|
+
export declare const readRequirements: (filePath: string) => Requirements;
|
|
@@ -4,7 +4,7 @@ import fs from 'fs';
|
|
|
4
4
|
* @param filePath
|
|
5
5
|
* @returns set of all dependencies names
|
|
6
6
|
*/
|
|
7
|
-
export const readRequirements = (filePath
|
|
7
|
+
export const readRequirements = (filePath) => {
|
|
8
8
|
const content = fs.readFileSync(filePath, 'utf8');
|
|
9
9
|
const lines = content.split('\n');
|
|
10
10
|
const requirements = {};
|
|
@@ -17,8 +17,8 @@ export const readRequirements = (filePath, describer) => {
|
|
|
17
17
|
// Extract package name (everything before version specifiers)
|
|
18
18
|
const packageMatch = trimmedLine.match(/^([a-zA-Z0-9_-]+)/);
|
|
19
19
|
if (packageMatch) {
|
|
20
|
-
const
|
|
21
|
-
requirements[
|
|
20
|
+
const packageName = packageMatch[1];
|
|
21
|
+
requirements[packageName] = trimmedLine;
|
|
22
22
|
}
|
|
23
23
|
}
|
|
24
24
|
return requirements;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Resolve the name of the import from the name of the package
|
|
3
3
|
* @param depName the package name
|
|
4
|
-
* @returns the import name
|
|
4
|
+
* @returns a map of the import name to the package name
|
|
5
5
|
*/
|
|
6
|
-
export declare const resolveDepNames: (depNames: string[], sitePackagesDir: string) =>
|
|
6
|
+
export declare const resolveDepNames: (depNames: string[], sitePackagesDir: string) => Record<string, string>;
|
|
@@ -4,13 +4,13 @@ import { PythonError } from './python-errors';
|
|
|
4
4
|
/**
|
|
5
5
|
* Resolve the name of the import from the name of the package
|
|
6
6
|
* @param depName the package name
|
|
7
|
-
* @returns the import name
|
|
7
|
+
* @returns a map of the import name to the package name
|
|
8
8
|
*/
|
|
9
9
|
export const resolveDepNames = (depNames, sitePackagesDir) => {
|
|
10
10
|
const folders = fs
|
|
11
11
|
.readdirSync(sitePackagesDir)
|
|
12
12
|
.filter((folder) => fs.statSync(path.join(sitePackagesDir, folder)).isDirectory());
|
|
13
|
-
const result =
|
|
13
|
+
const result = {};
|
|
14
14
|
for (const depName of depNames) {
|
|
15
15
|
const regex = new RegExp(`^${depName.replace(/-/g, '[-_]')}-.*\\.dist-info$`);
|
|
16
16
|
const folder = folders.find((folder) => regex.test(folder));
|
|
@@ -19,7 +19,7 @@ export const resolveDepNames = (depNames, sitePackagesDir) => {
|
|
|
19
19
|
const recordFile = fs.existsSync(path.join(sitePackagesDir, folder, 'RECORD'));
|
|
20
20
|
if (topLevelFile) {
|
|
21
21
|
const topLevel = fs.readFileSync(path.join(sitePackagesDir, folder, 'top_level.txt'), 'utf8').trim();
|
|
22
|
-
result
|
|
22
|
+
result[topLevel] = depName;
|
|
23
23
|
}
|
|
24
24
|
else if (recordFile) {
|
|
25
25
|
const record = fs.readFileSync(path.join(sitePackagesDir, folder, 'RECORD'), 'utf8').trim();
|
|
@@ -29,8 +29,7 @@ export const resolveDepNames = (depNames, sitePackagesDir) => {
|
|
|
29
29
|
// RECORD format: path/__init__.py,sha256=hash,size
|
|
30
30
|
const match = /^([^/]+)\/__init__\.py,/.exec(line);
|
|
31
31
|
if (match) {
|
|
32
|
-
result
|
|
33
|
-
break;
|
|
32
|
+
result[match[1]] = depName;
|
|
34
33
|
}
|
|
35
34
|
}
|
|
36
35
|
}
|
|
@@ -1,7 +1,6 @@
|
|
|
1
|
-
import { Requirements } from './read-requirements';
|
|
2
1
|
export type TraverseTreeResult = {
|
|
3
2
|
standardLibDependencies: Set<string>;
|
|
4
3
|
externalDependencies: Set<string>;
|
|
5
4
|
files: Set<string>;
|
|
6
5
|
};
|
|
7
|
-
export declare const traverseTree: (rootDir: string, filePath: string, result: TraverseTreeResult,
|
|
6
|
+
export declare const traverseTree: (rootDir: string, filePath: string, result: TraverseTreeResult, dependenciesMap: Record<string, string>, fileContent?: string) => void;
|
|
@@ -3,7 +3,7 @@ import path from 'path';
|
|
|
3
3
|
import { convertImportToPath } from './convert-import-path';
|
|
4
4
|
import { getDependenciesFromFile } from './get-dependencies-from-file';
|
|
5
5
|
import { PythonFileNotFoundError, PythonImportNotFoundError } from './python-errors';
|
|
6
|
-
export const traverseTree = (rootDir, filePath, result,
|
|
6
|
+
export const traverseTree = (rootDir, filePath, result, dependenciesMap,
|
|
7
7
|
// optional
|
|
8
8
|
fileContent) => {
|
|
9
9
|
const fileAbsolutePath = path.join(rootDir, filePath);
|
|
@@ -15,7 +15,7 @@ fileContent) => {
|
|
|
15
15
|
result.files.add(initPath.replace(rootDir, ''));
|
|
16
16
|
}
|
|
17
17
|
const content = fileContent || fs.readFileSync(fileAbsolutePath, 'utf8');
|
|
18
|
-
const dependencies = getDependenciesFromFile(content, filePath,
|
|
18
|
+
const dependencies = getDependenciesFromFile(content, filePath, dependenciesMap);
|
|
19
19
|
result.files.add(filePath);
|
|
20
20
|
dependencies.externalDependencies.forEach((dependency) => {
|
|
21
21
|
result.externalDependencies.add(dependency);
|
|
@@ -30,7 +30,7 @@ fileContent) => {
|
|
|
30
30
|
const dependencyPath = dependencyFilePath.replace(rootDir, '');
|
|
31
31
|
if (!result.files.has(dependencyPath)) {
|
|
32
32
|
try {
|
|
33
|
-
traverseTree(rootDir, dependencyPath, result,
|
|
33
|
+
traverseTree(rootDir, dependencyPath, result, dependenciesMap);
|
|
34
34
|
}
|
|
35
35
|
catch (error) {
|
|
36
36
|
if (error instanceof PythonFileNotFoundError) {
|
|
@@ -38,7 +38,7 @@ fileContent) => {
|
|
|
38
38
|
// try root folder
|
|
39
39
|
try {
|
|
40
40
|
const rootDependencyFilePath = path.resolve(rootDir, `${pythonPath}.py`).replace(rootDir, '');
|
|
41
|
-
return traverseTree(rootDir, rootDependencyFilePath, result,
|
|
41
|
+
return traverseTree(rootDir, rootDependencyFilePath, result, dependenciesMap);
|
|
42
42
|
}
|
|
43
43
|
catch (_error) {
|
|
44
44
|
// let it throw
|
|
@@ -6,7 +6,6 @@ export declare class PythonBuilder implements StepBuilder {
|
|
|
6
6
|
private readonly listener;
|
|
7
7
|
private packager;
|
|
8
8
|
constructor(builder: Builder, listener: BuildListener);
|
|
9
|
-
private getRequirements;
|
|
10
9
|
buildApiSteps(steps: Step<ApiRouteConfig>[]): Promise<RouterBuildResult>;
|
|
11
10
|
build(step: Step): Promise<void>;
|
|
12
11
|
private generatePackage;
|
|
@@ -4,4 +4,4 @@ export type PythonResult = {
|
|
|
4
4
|
externalDependencies: Requirements;
|
|
5
5
|
files: string[];
|
|
6
6
|
};
|
|
7
|
-
export declare const extractPythonData: (rootDir: string, filePath: string,
|
|
7
|
+
export declare const extractPythonData: (rootDir: string, filePath: string, dependenciesMap: Record<string, string>, fileContent?: string) => PythonResult;
|
|
@@ -1,7 +1,6 @@
|
|
|
1
|
-
import { Requirements } from './read-requirements';
|
|
2
1
|
export type Dependencies = {
|
|
3
2
|
standardLibDependencies: Set<string>;
|
|
4
3
|
externalDependencies: Set<string>;
|
|
5
4
|
projectDependencies: Set<string>;
|
|
6
5
|
};
|
|
7
|
-
export declare const getDependenciesFromFile: (content: string, path: string,
|
|
6
|
+
export declare const getDependenciesFromFile: (content: string, path: string, externalDependenciesMap: Record<string, string>) => Dependencies;
|
|
@@ -1,15 +1,13 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
type PackageDescriber = (packageName: string) => PackageInfo;
|
|
1
|
+
/**
|
|
2
|
+
* { libraryName: lineFromRequirementsFile }
|
|
3
|
+
*
|
|
4
|
+
* Example:
|
|
5
|
+
* { scikit-learn: 'scikit-learn==1.0.2' }
|
|
6
|
+
*/
|
|
7
|
+
export type Requirements = Record<string, string>;
|
|
9
8
|
/**
|
|
10
9
|
* Read the requirements.txt file and return a set of dependencies
|
|
11
10
|
* @param filePath
|
|
12
11
|
* @returns set of all dependencies names
|
|
13
12
|
*/
|
|
14
|
-
export declare const readRequirements: (filePath: string
|
|
15
|
-
export {};
|
|
13
|
+
export declare const readRequirements: (filePath: string) => Requirements;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Resolve the name of the import from the name of the package
|
|
3
3
|
* @param depName the package name
|
|
4
|
-
* @returns the import name
|
|
4
|
+
* @returns a map of the import name to the package name
|
|
5
5
|
*/
|
|
6
|
-
export declare const resolveDepNames: (depNames: string[], sitePackagesDir: string) =>
|
|
6
|
+
export declare const resolveDepNames: (depNames: string[], sitePackagesDir: string) => Record<string, string>;
|
|
@@ -1,7 +1,6 @@
|
|
|
1
|
-
import { Requirements } from './read-requirements';
|
|
2
1
|
export type TraverseTreeResult = {
|
|
3
2
|
standardLibDependencies: Set<string>;
|
|
4
3
|
externalDependencies: Set<string>;
|
|
5
4
|
files: Set<string>;
|
|
6
5
|
};
|
|
7
|
-
export declare const traverseTree: (rootDir: string, filePath: string, result: TraverseTreeResult,
|
|
6
|
+
export declare const traverseTree: (rootDir: string, filePath: string, result: TraverseTreeResult, dependenciesMap: Record<string, string>, fileContent?: string) => void;
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "motia",
|
|
3
3
|
"description": "A Modern Unified Backend Framework for APIs, Events and Agents",
|
|
4
|
-
"version": "0.7.2-beta.
|
|
4
|
+
"version": "0.7.2-beta.135-685562",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"repository": {
|
|
7
7
|
"type": "git",
|
|
@@ -46,9 +46,9 @@
|
|
|
46
46
|
"python-ast": "^0.1.0",
|
|
47
47
|
"table": "^6.9.0",
|
|
48
48
|
"ts-node": "^10.9.2",
|
|
49
|
-
"@motiadev/
|
|
50
|
-
"@motiadev/stream-client-node": "0.7.2-beta.
|
|
51
|
-
"@motiadev/
|
|
49
|
+
"@motiadev/core": "0.7.2-beta.135-685562",
|
|
50
|
+
"@motiadev/stream-client-node": "0.7.2-beta.135-685562",
|
|
51
|
+
"@motiadev/workbench": "0.7.2-beta.135-685562"
|
|
52
52
|
},
|
|
53
53
|
"devDependencies": {
|
|
54
54
|
"@amplitude/analytics-types": "^2.9.2",
|