nodepyx 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (184) hide show
  1. package/LICENSE +22 -0
  2. package/README.md +399 -0
  3. package/binding.gyp +73 -0
  4. package/dist/core/PyCallable.d.ts +65 -0
  5. package/dist/core/PyCallable.d.ts.map +1 -0
  6. package/dist/core/PyCallable.js +109 -0
  7. package/dist/core/PyCallable.js.map +1 -0
  8. package/dist/core/PyContext.d.ts +76 -0
  9. package/dist/core/PyContext.d.ts.map +1 -0
  10. package/dist/core/PyContext.js +228 -0
  11. package/dist/core/PyContext.js.map +1 -0
  12. package/dist/core/PyIterator.d.ts +84 -0
  13. package/dist/core/PyIterator.d.ts.map +1 -0
  14. package/dist/core/PyIterator.js +243 -0
  15. package/dist/core/PyIterator.js.map +1 -0
  16. package/dist/core/PyModule.d.ts +55 -0
  17. package/dist/core/PyModule.d.ts.map +1 -0
  18. package/dist/core/PyModule.js +172 -0
  19. package/dist/core/PyModule.js.map +1 -0
  20. package/dist/core/PyProxy.d.ts +65 -0
  21. package/dist/core/PyProxy.d.ts.map +1 -0
  22. package/dist/core/PyProxy.js +483 -0
  23. package/dist/core/PyProxy.js.map +1 -0
  24. package/dist/core/PyRuntime.d.ts +105 -0
  25. package/dist/core/PyRuntime.d.ts.map +1 -0
  26. package/dist/core/PyRuntime.js +438 -0
  27. package/dist/core/PyRuntime.js.map +1 -0
  28. package/dist/env/CondaManager.d.ts +118 -0
  29. package/dist/env/CondaManager.d.ts.map +1 -0
  30. package/dist/env/CondaManager.js +401 -0
  31. package/dist/env/CondaManager.js.map +1 -0
  32. package/dist/env/PackageInstaller.d.ts +233 -0
  33. package/dist/env/PackageInstaller.d.ts.map +1 -0
  34. package/dist/env/PackageInstaller.js +609 -0
  35. package/dist/env/PackageInstaller.js.map +1 -0
  36. package/dist/env/PythonDetector.d.ts +103 -0
  37. package/dist/env/PythonDetector.d.ts.map +1 -0
  38. package/dist/env/PythonDetector.js +381 -0
  39. package/dist/env/PythonDetector.js.map +1 -0
  40. package/dist/env/VenvManager.d.ts +117 -0
  41. package/dist/env/VenvManager.d.ts.map +1 -0
  42. package/dist/env/VenvManager.js +331 -0
  43. package/dist/env/VenvManager.js.map +1 -0
  44. package/dist/index.d.ts +169 -0
  45. package/dist/index.d.ts.map +1 -0
  46. package/dist/index.js +393 -0
  47. package/dist/index.js.map +1 -0
  48. package/dist/plugins/Plugin.interface.d.ts +41 -0
  49. package/dist/plugins/Plugin.interface.d.ts.map +1 -0
  50. package/dist/plugins/Plugin.interface.js +12 -0
  51. package/dist/plugins/Plugin.interface.js.map +1 -0
  52. package/dist/plugins/PluginManager.d.ts +26 -0
  53. package/dist/plugins/PluginManager.d.ts.map +1 -0
  54. package/dist/plugins/PluginManager.js +174 -0
  55. package/dist/plugins/PluginManager.js.map +1 -0
  56. package/dist/plugins/builtin/NumpyPlugin.d.ts +17 -0
  57. package/dist/plugins/builtin/NumpyPlugin.d.ts.map +1 -0
  58. package/dist/plugins/builtin/NumpyPlugin.js +41 -0
  59. package/dist/plugins/builtin/NumpyPlugin.js.map +1 -0
  60. package/dist/plugins/builtin/PandasPlugin.d.ts +19 -0
  61. package/dist/plugins/builtin/PandasPlugin.d.ts.map +1 -0
  62. package/dist/plugins/builtin/PandasPlugin.js +57 -0
  63. package/dist/plugins/builtin/PandasPlugin.js.map +1 -0
  64. package/dist/plugins/builtin/TorchPlugin.d.ts +23 -0
  65. package/dist/plugins/builtin/TorchPlugin.d.ts.map +1 -0
  66. package/dist/plugins/builtin/TorchPlugin.js +50 -0
  67. package/dist/plugins/builtin/TorchPlugin.js.map +1 -0
  68. package/dist/plugins/index.d.ts +7 -0
  69. package/dist/plugins/index.d.ts.map +1 -0
  70. package/dist/plugins/index.js +12 -0
  71. package/dist/plugins/index.js.map +1 -0
  72. package/dist/serialization/DataFrameBridge.d.ts +141 -0
  73. package/dist/serialization/DataFrameBridge.d.ts.map +1 -0
  74. package/dist/serialization/DataFrameBridge.js +355 -0
  75. package/dist/serialization/DataFrameBridge.js.map +1 -0
  76. package/dist/serialization/MsgPackSerializer.d.ts +45 -0
  77. package/dist/serialization/MsgPackSerializer.d.ts.map +1 -0
  78. package/dist/serialization/MsgPackSerializer.js +242 -0
  79. package/dist/serialization/MsgPackSerializer.js.map +1 -0
  80. package/dist/serialization/NumpyBridge.d.ts +96 -0
  81. package/dist/serialization/NumpyBridge.d.ts.map +1 -0
  82. package/dist/serialization/NumpyBridge.js +323 -0
  83. package/dist/serialization/NumpyBridge.js.map +1 -0
  84. package/dist/serialization/Serializer.d.ts +78 -0
  85. package/dist/serialization/Serializer.d.ts.map +1 -0
  86. package/dist/serialization/Serializer.js +281 -0
  87. package/dist/serialization/Serializer.js.map +1 -0
  88. package/dist/types/PythonTypeMapper.d.ts +87 -0
  89. package/dist/types/PythonTypeMapper.d.ts.map +1 -0
  90. package/dist/types/PythonTypeMapper.js +449 -0
  91. package/dist/types/PythonTypeMapper.js.map +1 -0
  92. package/dist/types/StubCache.d.ts +109 -0
  93. package/dist/types/StubCache.d.ts.map +1 -0
  94. package/dist/types/StubCache.js +333 -0
  95. package/dist/types/StubCache.js.map +1 -0
  96. package/dist/types/TypeGenerator.d.ts +139 -0
  97. package/dist/types/TypeGenerator.d.ts.map +1 -0
  98. package/dist/types/TypeGenerator.js +372 -0
  99. package/dist/types/TypeGenerator.js.map +1 -0
  100. package/dist/types/addon.d.ts +114 -0
  101. package/dist/types/addon.d.ts.map +1 -0
  102. package/dist/types/addon.js +32 -0
  103. package/dist/types/addon.js.map +1 -0
  104. package/dist/types/config.d.ts +175 -0
  105. package/dist/types/config.d.ts.map +1 -0
  106. package/dist/types/config.js +35 -0
  107. package/dist/types/config.js.map +1 -0
  108. package/dist/types/index.d.ts +10 -0
  109. package/dist/types/index.d.ts.map +1 -0
  110. package/dist/types/index.js +12 -0
  111. package/dist/types/index.js.map +1 -0
  112. package/dist/types/python.d.ts +235 -0
  113. package/dist/types/python.d.ts.map +1 -0
  114. package/dist/types/python.js +7 -0
  115. package/dist/types/python.js.map +1 -0
  116. package/dist/utils/ErrorTranslator.d.ts +83 -0
  117. package/dist/utils/ErrorTranslator.d.ts.map +1 -0
  118. package/dist/utils/ErrorTranslator.js +210 -0
  119. package/dist/utils/ErrorTranslator.js.map +1 -0
  120. package/dist/utils/Logger.d.ts +27 -0
  121. package/dist/utils/Logger.d.ts.map +1 -0
  122. package/dist/utils/Logger.js +115 -0
  123. package/dist/utils/Logger.js.map +1 -0
  124. package/dist/utils/MemoryMonitor.d.ts +44 -0
  125. package/dist/utils/MemoryMonitor.d.ts.map +1 -0
  126. package/dist/utils/MemoryMonitor.js +143 -0
  127. package/dist/utils/MemoryMonitor.js.map +1 -0
  128. package/package.json +177 -0
  129. package/python/error_handler.py +433 -0
  130. package/python/nodepyx_runtime.py +575 -0
  131. package/python/serializer.py +379 -0
  132. package/python/type_inspector.py +288 -0
  133. package/scripts/build-native.js +68 -0
  134. package/scripts/download-prebuilds.js +99 -0
  135. package/scripts/generate-stubs.js +405 -0
  136. package/scripts/install.js +260 -0
  137. package/src/core/PyCallable.ts +137 -0
  138. package/src/core/PyContext.ts +296 -0
  139. package/src/core/PyIterator.ts +294 -0
  140. package/src/core/PyModule.ts +194 -0
  141. package/src/core/PyProxy.ts +605 -0
  142. package/src/core/PyRuntime.ts +504 -0
  143. package/src/env/CondaManager.ts +451 -0
  144. package/src/env/PackageInstaller.ts +738 -0
  145. package/src/env/PythonDetector.ts +414 -0
  146. package/src/env/VenvManager.ts +396 -0
  147. package/src/index.ts +425 -0
  148. package/src/native/gil_guard.cpp +26 -0
  149. package/src/native/gil_guard.h +175 -0
  150. package/src/native/nodepyx_addon.cpp +886 -0
  151. package/src/native/python_bridge.cpp +790 -0
  152. package/src/native/python_bridge.h +257 -0
  153. package/src/native/thread_pool.cpp +336 -0
  154. package/src/native/thread_pool.h +175 -0
  155. package/src/native/type_converter.cpp +901 -0
  156. package/src/native/type_converter.h +272 -0
  157. package/src/nextjs/PyProvider.tsx +123 -0
  158. package/src/nextjs/index.ts +21 -0
  159. package/src/nextjs/usePython.ts +106 -0
  160. package/src/nextjs/withnodepyx.ts +88 -0
  161. package/src/plugins/Plugin.interface.ts +51 -0
  162. package/src/plugins/PluginManager.ts +155 -0
  163. package/src/plugins/builtin/NumpyPlugin.ts +36 -0
  164. package/src/plugins/builtin/PandasPlugin.ts +49 -0
  165. package/src/plugins/builtin/TorchPlugin.ts +56 -0
  166. package/src/plugins/index.ts +7 -0
  167. package/src/serialization/DataFrameBridge.ts +398 -0
  168. package/src/serialization/MsgPackSerializer.ts +220 -0
  169. package/src/serialization/NumpyBridge.ts +332 -0
  170. package/src/serialization/Serializer.ts +320 -0
  171. package/src/types/PythonTypeMapper.ts +495 -0
  172. package/src/types/StubCache.ts +340 -0
  173. package/src/types/TypeGenerator.ts +491 -0
  174. package/src/types/addon.ts +170 -0
  175. package/src/types/config.ts +226 -0
  176. package/src/types/index.ts +55 -0
  177. package/src/types/python.ts +309 -0
  178. package/src/types/stubs/numpy.d.ts +441 -0
  179. package/src/types/stubs/pandas.d.ts +575 -0
  180. package/src/types/stubs/sklearn.d.ts +728 -0
  181. package/src/types/stubs/torch.d.ts +694 -0
  182. package/src/utils/ErrorTranslator.ts +220 -0
  183. package/src/utils/Logger.ts +119 -0
  184. package/src/utils/MemoryMonitor.ts +175 -0
@@ -0,0 +1,738 @@
1
+ /**
2
+ * nodepyx — PackageInstaller
3
+ *
4
+ * Standalone pip-based package installer that works both inside and outside
5
+ * virtualenv/conda contexts. It is the single place where all pip interaction
6
+ * is concentrated so that VenvManager, CondaManager, and the public nodepyx API
7
+ * all delegate here instead of shelling out to pip themselves.
8
+ *
9
+ * Responsibilities
10
+ * ──────────────────────────────────────────────────────────────────────────────
11
+ * • Resolve the "import name" from the "install name" (e.g. scikit-learn → sklearn)
12
+ * • Check whether a package is already importable before trying to install it
13
+ * • Run pip with the correct flags (--quiet, --no-warn-script-location, etc.)
14
+ * • Support per-package version constraints ("numpy>=1.24,<2")
15
+ * • Support extras ("uvicorn[standard]")
16
+ * • Support index overrides (corporate PyPI mirrors, --extra-index-url)
17
+ * • Upgrade existing installations when requested
18
+ * • Emit structured progress events via EventEmitter
19
+ * • Retry transient network failures up to N times with back-off
20
+ * • Parse pip's JSON output (--report) for installed metadata when available
21
+ * • Return full InstallResult objects describing what happened
22
+ *
23
+ * Usage
24
+ * ──────────────────────────────────────────────────────────────────────────────
25
+ * const installer = new PackageInstaller({ pythonExecutable: '/usr/bin/python3' });
26
+ *
27
+ * const result = await installer.install(['pandas', 'numpy>=1.24', 'torch']);
28
+ * console.log(result.installed); // ['pandas==2.2.2', 'numpy==1.26.4', ...]
29
+ *
30
+ * installer.on('progress', (ev) => console.log(ev.package, ev.status));
31
+ */
32
+
33
+ import { EventEmitter } from 'events';
34
+ import { spawnSync, spawn } from 'child_process';
35
+ import * as fs from 'fs';
36
+ import * as path from 'path';
37
+ import { Logger } from '../utils/Logger';
38
+
39
+ const logger = new Logger('PackageInstaller');
40
+
41
+ // ─── Public types ──────────────────────────────────────────────────────────────
42
+
43
+ /** A single package specification that can be installed. */
44
+ export interface PackageSpec {
45
+ /** Raw pip spec: "pandas", "numpy>=1.24,<2", "torch==2.3.0+cu121" */
46
+ spec: string;
47
+ /**
48
+ * Override the import (module) name used to test whether the package is
49
+ * already importable. Defaults to the auto-detected mapping.
50
+ * e.g. spec="scikit-learn" → importName="sklearn"
51
+ */
52
+ importName?: string;
53
+ /** Force reinstall even if already importable. */
54
+ forceReinstall?: boolean;
55
+ /** Install pre-release versions */
56
+ preRelease?: boolean;
57
+ }
58
+
59
+ /** Options passed to the PackageInstaller constructor. */
60
+ export interface PackageInstallerOptions {
61
+ /** Python executable to use for pip invocations. Required. */
62
+ pythonExecutable: string;
63
+ /**
64
+ * Extra --index-url override (corporate mirror etc.)
65
+ * If omitted, pip uses its default index.
66
+ */
67
+ indexUrl?: string;
68
+ /**
69
+ * Additional --extra-index-url entries (e.g. CUDA wheels).
70
+ * Supports multiple URLs.
71
+ */
72
+ extraIndexUrls?: string[];
73
+ /**
74
+ * Trusted hosts (--trusted-host) required when using HTTP mirrors.
75
+ */
76
+ trustedHosts?: string[];
77
+ /**
78
+ * Maximum retry attempts for transient network errors. Default: 3.
79
+ */
80
+ maxRetries?: number;
81
+ /**
82
+ * Base delay (ms) between retry attempts. Each retry doubles. Default: 1000.
83
+ */
84
+ retryDelayMs?: number;
85
+ /**
86
+ * Working directory for pip. Defaults to process.cwd().
87
+ */
88
+ cwd?: string;
89
+ /**
90
+ * Additional environment variables to inject into pip subprocess.
91
+ */
92
+ env?: Record<string, string>;
93
+ /**
94
+ * Timeout (ms) per pip invocation. Default: 300 000 (5 min).
95
+ */
96
+ timeoutMs?: number;
97
+ /**
98
+ * If true, pass --quiet to suppress pip chatter. Default: true.
99
+ */
100
+ quiet?: boolean;
101
+ /**
102
+ * If true, pass --upgrade to pip for already-installed packages. Default: false.
103
+ */
104
+ upgrade?: boolean;
105
+ }
106
+
107
+ /** Status of a single package during/after installation. */
108
+ export type PackageStatus =
109
+ | 'already_installed'
110
+ | 'installing'
111
+ | 'installed'
112
+ | 'upgraded'
113
+ | 'failed'
114
+ | 'skipped';
115
+
116
+ /** Progress event emitted during installation. */
117
+ export interface InstallProgressEvent {
118
+ package: string;
119
+ status: PackageStatus;
120
+ version?: string;
121
+ error?: string;
122
+ }
123
+
124
+ /** Aggregated result of an install() call. */
125
+ export interface InstallResult {
126
+ /** Packages that were successfully installed or upgraded (with versions). */
127
+ installed: string[];
128
+ /** Packages that were already present and did not need installation. */
129
+ alreadyInstalled: string[];
130
+ /** Packages that failed to install. */
131
+ failed: Array<{ spec: string; error: string }>;
132
+ /** Whether every requested package ended up available. */
133
+ success: boolean;
134
+ /** Combined stdout/stderr from pip (useful for debugging). */
135
+ output: string;
136
+ }
137
+
138
+ /** Metadata returned by `pip show`. */
139
+ export interface PackageMetadata {
140
+ name: string;
141
+ version: string;
142
+ location: string;
143
+ requires: string[];
144
+ requiredBy: string[];
145
+ summary: string;
146
+ }
147
+
148
+ // ─── Import-name alias table ──────────────────────────────────────────────────
149
+ // Maps pip install name → Python import name.
150
+ // Only entries that differ from the install name need to be listed here.
151
+
152
+ const INSTALL_TO_IMPORT: Record<string, string> = {
153
+ 'scikit-learn': 'sklearn',
154
+ 'scikit_learn': 'sklearn',
155
+ 'Pillow': 'PIL',
156
+ 'pillow': 'PIL',
157
+ 'opencv-python': 'cv2',
158
+ 'opencv-python-headless': 'cv2',
159
+ 'opencv_python': 'cv2',
160
+ 'beautifulsoup4': 'bs4',
161
+ 'beautifulsoup': 'bs4',
162
+ 'pyyaml': 'yaml',
163
+ 'PyYAML': 'yaml',
164
+ 'python-dateutil': 'dateutil',
165
+ 'python_dateutil': 'dateutil',
166
+ 'typing-extensions': 'typing_extensions',
167
+ 'typing_extensions': 'typing_extensions',
168
+ 'pyzmq': 'zmq',
169
+ 'Pyzmq': 'zmq',
170
+ 'protobuf': 'google.protobuf',
171
+ 'grpcio': 'grpc',
172
+ 'psutil': 'psutil',
173
+ 'python-dotenv': 'dotenv',
174
+ 'python_dotenv': 'dotenv',
175
+ 'Werkzeug': 'werkzeug',
176
+ 'Flask': 'flask',
177
+ 'Django': 'django',
178
+ 'Jinja2': 'jinja2',
179
+ 'SQLAlchemy': 'sqlalchemy',
180
+ 'alembic': 'alembic',
181
+ 'aiohttp': 'aiohttp',
182
+ 'httpx': 'httpx',
183
+ 'fastapi': 'fastapi',
184
+ 'starlette': 'starlette',
185
+ 'uvicorn': 'uvicorn',
186
+ 'gunicorn': 'gunicorn',
187
+ 'celery': 'celery',
188
+ 'redis': 'redis',
189
+ 'pymongo': 'pymongo',
190
+ 'motor': 'motor',
191
+ 'aiomysql': 'aiomysql',
192
+ 'asyncpg': 'asyncpg',
193
+ 'psycopg2': 'psycopg2',
194
+ 'psycopg2-binary': 'psycopg2',
195
+ 'psycopg2_binary': 'psycopg2',
196
+ 'boto3': 'boto3',
197
+ 'botocore': 'botocore',
198
+ 'google-cloud-storage': 'google.cloud.storage',
199
+ 'google-auth': 'google.auth',
200
+ 'azure-storage-blob': 'azure.storage.blob',
201
+ 'minio': 'minio',
202
+ 'transformers': 'transformers',
203
+ 'datasets': 'datasets',
204
+ 'tokenizers': 'tokenizers',
205
+ 'diffusers': 'diffusers',
206
+ 'accelerate': 'accelerate',
207
+ 'peft': 'peft',
208
+ 'trl': 'trl',
209
+ 'sentence-transformers': 'sentence_transformers',
210
+ 'sentence_transformers': 'sentence_transformers',
211
+ 'faiss-cpu': 'faiss',
212
+ 'faiss-gpu': 'faiss',
213
+ 'annoy': 'annoy',
214
+ 'chromadb': 'chromadb',
215
+ 'pinecone-client': 'pinecone',
216
+ 'weaviate-client': 'weaviate',
217
+ 'langchain': 'langchain',
218
+ 'openai': 'openai',
219
+ 'anthropic': 'anthropic',
220
+ 'tiktoken': 'tiktoken',
221
+ 'matplotlib': 'matplotlib',
222
+ 'seaborn': 'seaborn',
223
+ 'plotly': 'plotly',
224
+ 'bokeh': 'bokeh',
225
+ 'altair': 'altair',
226
+ 'scipy': 'scipy',
227
+ 'statsmodels': 'statsmodels',
228
+ 'lightgbm': 'lightgbm',
229
+ 'xgboost': 'xgboost',
230
+ 'catboost': 'catboost',
231
+ 'shap': 'shap',
232
+ 'lime': 'lime',
233
+ 'sympy': 'sympy',
234
+ 'networkx': 'networkx',
235
+ 'pyarrow': 'pyarrow',
236
+ 'pyarrow[flight]': 'pyarrow',
237
+ 'polars': 'polars',
238
+ 'dask': 'dask',
239
+ 'ray': 'ray',
240
+ 'pydantic': 'pydantic',
241
+ 'pydantic-settings': 'pydantic_settings',
242
+ 'attrs': 'attr',
243
+ 'cattrs': 'cattr',
244
+ 'click': 'click',
245
+ 'typer': 'typer',
246
+ 'rich': 'rich',
247
+ 'tqdm': 'tqdm',
248
+ 'loguru': 'loguru',
249
+ 'structlog': 'structlog',
250
+ 'pendulum': 'pendulum',
251
+ 'arrow': 'arrow',
252
+ 'humanize': 'humanize',
253
+ 'tabulate': 'tabulate',
254
+ 'prettytable': 'prettytable',
255
+ 'orjson': 'orjson',
256
+ 'ujson': 'ujson',
257
+ 'msgpack': 'msgpack',
258
+ 'lz4': 'lz4',
259
+ 'zstandard': 'zstd',
260
+ 'cryptography': 'cryptography',
261
+ 'paramiko': 'paramiko',
262
+ 'fabric': 'fabric',
263
+ 'invoke': 'invoke',
264
+ 'pytest': 'pytest',
265
+ 'hypothesis': 'hypothesis',
266
+ 'mock': 'unittest.mock',
267
+ 'freezegun': 'freezegun',
268
+ 'factory-boy': 'factory',
269
+ 'Faker': 'faker',
270
+ };
271
+
272
+ // ─── Helpers ───────────────────────────────────────────────────────────────────
273
+
274
+ /**
275
+ * Extract the bare package name from a spec string.
276
+ * "numpy>=1.24,<2" → "numpy"
277
+ * "torch==2.3.0+cu121" → "torch"
278
+ * "uvicorn[standard]" → "uvicorn"
279
+ */
280
+ function extractPackageName(spec: string): string {
281
+ // Remove VCS prefixes (git+https://...)
282
+ if (spec.startsWith('git+') || spec.startsWith('hg+') || spec.startsWith('svn+')) {
283
+ const eggIndex = spec.indexOf('#egg=');
284
+ if (eggIndex !== -1) {return spec.slice(eggIndex + 5).split('&')[0]!.trim();}
285
+ return spec;
286
+ }
287
+ // Remove extras [...], then split on version specifiers
288
+ return spec.replace(/\[.*?\]/g, '').split(/[><=!~;]/)[0]!.trim();
289
+ }
290
+
291
+ /**
292
+ * Determine the Python import name for a given install spec.
293
+ */
294
+ function resolveImportName(spec: string, override?: string): string {
295
+ if (override) {return override;}
296
+ const bare = extractPackageName(spec);
297
+ return INSTALL_TO_IMPORT[bare] ?? bare.replace(/-/g, '_');
298
+ }
299
+
300
+ /** Escape a string for safe inclusion in a Python one-liner. */
301
+ function escapePy(s: string): string {
302
+ return s.replace(/\\/g, '\\\\').replace(/'/g, "\\'");
303
+ }
304
+
305
+ /** Sleep helper. */
306
+ function sleep(ms: number): Promise<void> {
307
+ return new Promise((r) => setTimeout(r, ms));
308
+ }
309
+
310
+ // ─── PackageInstaller ──────────────────────────────────────────────────────────
311
+
312
+ export class PackageInstaller extends EventEmitter {
313
+ private readonly _python: string;
314
+ private readonly _opts: Required<PackageInstallerOptions>;
315
+
316
+ constructor(options: PackageInstallerOptions) {
317
+ super();
318
+ this._python = options.pythonExecutable;
319
+ this._opts = {
320
+ pythonExecutable: options.pythonExecutable,
321
+ indexUrl: options.indexUrl ?? '',
322
+ extraIndexUrls: options.extraIndexUrls ?? [],
323
+ trustedHosts: options.trustedHosts ?? [],
324
+ maxRetries: options.maxRetries ?? 3,
325
+ retryDelayMs: options.retryDelayMs ?? 1000,
326
+ cwd: options.cwd ?? process.cwd(),
327
+ env: options.env ?? {},
328
+ timeoutMs: options.timeoutMs ?? 300_000,
329
+ quiet: options.quiet !== false,
330
+ upgrade: options.upgrade ?? false,
331
+ };
332
+ }
333
+
334
+ // ── Public API ──────────────────────────────────────────────────────────────
335
+
336
+ /**
337
+ * Install one or more packages.
338
+ *
339
+ * @param packages - Array of pip spec strings or PackageSpec objects.
340
+ * @returns InstallResult describing what happened to each package.
341
+ */
342
+ async install(packages: Array<string | PackageSpec>): Promise<InstallResult> {
343
+ const specs = packages.map((p) =>
344
+ typeof p === 'string' ? { spec: p } as PackageSpec : p
345
+ );
346
+
347
+ const result: InstallResult = {
348
+ installed: [],
349
+ alreadyInstalled: [],
350
+ failed: [],
351
+ success: false,
352
+ output: '',
353
+ };
354
+
355
+ const toInstall: PackageSpec[] = [];
356
+
357
+ // ── 1. Pre-flight: check which are already importable ──────────────────
358
+ for (const spec of specs) {
359
+ const importName = resolveImportName(spec.spec, spec.importName);
360
+ if (!spec.forceReinstall && !this._opts.upgrade && this._isImportable(importName)) {
361
+ logger.debug(`Already importable: ${importName} (${spec.spec})`);
362
+ result.alreadyInstalled.push(spec.spec);
363
+ this._emit('already_installed', spec.spec);
364
+ } else {
365
+ toInstall.push(spec);
366
+ }
367
+ }
368
+
369
+ if (toInstall.length === 0) {
370
+ result.success = true;
371
+ return result;
372
+ }
373
+
374
+ // ── 2. Install in a single pip invocation where possible ──────────────
375
+ // Group specs that do not need individual treatment
376
+ const batchSpecs = toInstall.filter((s) => !s.preRelease);
377
+ const prerelSpecs = toInstall.filter((s) => s.preRelease);
378
+
379
+ const outputs: string[] = [];
380
+
381
+ if (batchSpecs.length > 0) {
382
+ const { out, perPackageResults } = await this._pipInstall(batchSpecs, false);
383
+ outputs.push(out);
384
+ this._mergeResults(result, perPackageResults);
385
+ }
386
+
387
+ if (prerelSpecs.length > 0) {
388
+ const { out, perPackageResults } = await this._pipInstall(prerelSpecs, true);
389
+ outputs.push(out);
390
+ this._mergeResults(result, perPackageResults);
391
+ }
392
+
393
+ result.output = outputs.join('\n');
394
+ result.success = result.failed.length === 0;
395
+ return result;
396
+ }
397
+
398
+ /**
399
+ * Install a single package and return the installed version string.
400
+ * Throws on failure.
401
+ */
402
+ async installOne(spec: string, options?: Partial<PackageSpec>): Promise<string> {
403
+ const result = await this.install([{ spec, ...options }]);
404
+ if (!result.success) {
405
+ const err = result.failed[0];
406
+ throw new Error(`Failed to install ${spec}: ${err?.error ?? 'unknown error'}`);
407
+ }
408
+ return this.getVersion(extractPackageName(spec)) ?? spec;
409
+ }
410
+
411
+ /**
412
+ * Upgrade a package to the latest version.
413
+ */
414
+ async upgrade(spec: string): Promise<string> {
415
+ return this.installOne(spec, { forceReinstall: false });
416
+ }
417
+
418
+ /**
419
+ * Uninstall one or more packages.
420
+ */
421
+ async uninstall(packages: string[]): Promise<void> {
422
+ if (packages.length === 0) {return;}
423
+
424
+ const args = ['-m', 'pip', 'uninstall', '-y', ...packages];
425
+ logger.info(`Uninstalling: ${packages.join(', ')}`);
426
+
427
+ const result = spawnSync(this._python, args, {
428
+ encoding: 'utf8',
429
+ cwd: this._opts.cwd,
430
+ env: { ...process.env, ...this._opts.env },
431
+ });
432
+
433
+ if (result.status !== 0) {
434
+ throw new Error(`pip uninstall failed:\n${result.stderr}`);
435
+ }
436
+ }
437
+
438
+ /**
439
+ * Check whether a package is currently importable by Python.
440
+ */
441
+ isImportable(importName: string): boolean {
442
+ return this._isImportable(importName);
443
+ }
444
+
445
+ /**
446
+ * Check whether a package (by install name) is installed and return
447
+ * its version string, or null if not installed.
448
+ */
449
+ getVersion(packageName: string): string | null {
450
+ const args = ['-m', 'pip', 'show', '--quiet', packageName];
451
+ const r = spawnSync(this._python, args, {
452
+ encoding: 'utf8',
453
+ env: { ...process.env, ...this._opts.env },
454
+ });
455
+ if (r.status !== 0 || !r.stdout) {return null;}
456
+ const match = r.stdout.match(/^Version:\s*(.+)$/m);
457
+ return match ? match[1]!.trim() : null;
458
+ }
459
+
460
+ /**
461
+ * Return full metadata for an installed package.
462
+ */
463
+ getMetadata(packageName: string): PackageMetadata | null {
464
+ const args = ['-m', 'pip', 'show', packageName];
465
+ const r = spawnSync(this._python, args, {
466
+ encoding: 'utf8',
467
+ env: { ...process.env, ...this._opts.env },
468
+ });
469
+ if (r.status !== 0 || !r.stdout) {return null;}
470
+ return this._parsePipShow(r.stdout);
471
+ }
472
+
473
+ /**
474
+ * List all packages installed in the current Python environment.
475
+ */
476
+ listInstalled(): Array<{ name: string; version: string }> {
477
+ const args = ['-m', 'pip', 'list', '--format=json'];
478
+ const r = spawnSync(this._python, args, {
479
+ encoding: 'utf8',
480
+ env: { ...process.env, ...this._opts.env },
481
+ });
482
+ if (r.status !== 0 || !r.stdout) {return [];}
483
+ try {
484
+ return JSON.parse(r.stdout) as Array<{ name: string; version: string }>;
485
+ } catch {
486
+ return [];
487
+ }
488
+ }
489
+
490
+ /**
491
+ * Check which packages from the given list are missing.
492
+ *
493
+ * @param packages - Array of pip spec strings.
494
+ * @returns Array of spec strings that are NOT currently importable.
495
+ */
496
+ getMissing(packages: string[]): string[] {
497
+ return packages.filter((spec) => {
498
+ const importName = resolveImportName(spec);
499
+ return !this._isImportable(importName);
500
+ });
501
+ }
502
+
503
+ /**
504
+ * Ensure pip itself is up-to-date.
505
+ */
506
+ async upgradePip(): Promise<void> {
507
+ logger.info('Upgrading pip...');
508
+ const args = ['-m', 'pip', 'install', '--upgrade', 'pip'];
509
+ const r = spawnSync(this._python, args, {
510
+ encoding: 'utf8',
511
+ cwd: this._opts.cwd,
512
+ env: { ...process.env, ...this._opts.env },
513
+ });
514
+ if (r.status !== 0) {
515
+ logger.warn(`pip self-upgrade failed (non-fatal):\n${r.stderr}`);
516
+ }
517
+ }
518
+
519
+ // ── Private helpers ────────────────────────────────────────────────────────
520
+
521
+ /**
522
+ * Run `python -c "import X"` to test importability.
523
+ * This is faster than `pip show` for the common case.
524
+ */
525
+ private _isImportable(importName: string): boolean {
526
+ // Handle dotted names like "google.protobuf"
527
+ const topLevel = importName.split('.')[0]!;
528
+ const code = `import ${escapePy(topLevel)}`;
529
+ const r = spawnSync(this._python, ['-c', code], {
530
+ encoding: 'utf8',
531
+ env: { ...process.env, ...this._opts.env },
532
+ });
533
+ return r.status === 0;
534
+ }
535
+
536
+ /**
537
+ * Build pip install argument list.
538
+ */
539
+ private _buildPipArgs(specs: PackageSpec[], preRelease: boolean): string[] {
540
+ const args: string[] = ['-m', 'pip', 'install'];
541
+
542
+ if (this._opts.quiet) {args.push('--quiet');}
543
+ if (this._opts.upgrade) {args.push('--upgrade');}
544
+ if (preRelease) {args.push('--pre');}
545
+
546
+ args.push('--no-warn-script-location');
547
+ args.push('--disable-pip-version-check');
548
+
549
+ if (this._opts.indexUrl) {
550
+ args.push('--index-url', this._opts.indexUrl);
551
+ }
552
+ for (const url of this._opts.extraIndexUrls) {
553
+ args.push('--extra-index-url', url);
554
+ }
555
+ for (const host of this._opts.trustedHosts) {
556
+ args.push('--trusted-host', host);
557
+ }
558
+
559
+ for (const s of specs) {
560
+ if (s.forceReinstall) {args.push('--force-reinstall');}
561
+ args.push(s.spec);
562
+ }
563
+
564
+ return args;
565
+ }
566
+
567
+ /**
568
+ * Execute pip install with retry logic.
569
+ */
570
+ private async _pipInstall(
571
+ specs: PackageSpec[],
572
+ preRelease: boolean
573
+ ): Promise<{ out: string; perPackageResults: Array<{ spec: string; status: PackageStatus; version?: string; error?: string }> }> {
574
+ const args = this._buildPipArgs(specs, preRelease);
575
+ const specList = specs.map((s) => s.spec).join(', ');
576
+
577
+ logger.info(`pip install: ${specList}`);
578
+ specs.forEach((s) => this._emit('installing', s.spec));
579
+
580
+ let lastError = '';
581
+ let lastOut = '';
582
+
583
+ for (let attempt = 1; attempt <= this._opts.maxRetries; attempt++) {
584
+ const { exitCode, stdout } = await this._runPip(args);
585
+ lastOut = stdout;
586
+
587
+ if (exitCode === 0) {
588
+ // Success — resolve versions
589
+ const results = specs.map((s) => {
590
+ const name = extractPackageName(s.spec);
591
+ const version = this.getVersion(name) ?? undefined;
592
+ this._emit('installed', s.spec, version);
593
+ return { spec: s.spec, status: 'installed' as PackageStatus, version };
594
+ });
595
+ return { out: stdout, perPackageResults: results };
596
+ }
597
+
598
+ lastError = stdout;
599
+ const isNetworkError = /connection error|timed out|network/i.test(stdout);
600
+
601
+ if (!isNetworkError || attempt === this._opts.maxRetries) {
602
+ break;
603
+ }
604
+
605
+ const delay = this._opts.retryDelayMs * 2 ** (attempt - 1);
606
+ logger.warn(`pip attempt ${attempt} failed (network). Retrying in ${delay}ms…`);
607
+ await sleep(delay);
608
+ }
609
+
610
+ // Failure path — try each spec individually to isolate the bad one
611
+ const results: Array<{ spec: string; status: PackageStatus; version?: string; error?: string }> = [];
612
+
613
+ if (specs.length > 1) {
614
+ logger.warn('Batch install failed — falling back to per-package install');
615
+ for (const s of specs) {
616
+ const single = await this._pipInstall([s], preRelease);
617
+ results.push(...single.perPackageResults);
618
+ }
619
+ } else {
620
+ const s = specs[0]!;
621
+ logger.error(`Failed to install ${s.spec}: ${lastError.slice(0, 200)}`);
622
+ this._emit('failed', s.spec, undefined, lastError);
623
+ results.push({ spec: s.spec, status: 'failed', error: lastError });
624
+ }
625
+
626
+ return { out: lastOut, perPackageResults: results };
627
+ }
628
+
629
+ /**
630
+ * Spawn pip and collect output.
631
+ */
632
+ private _runPip(args: string[]): Promise<{ exitCode: number; stdout: string }> {
633
+ return new Promise((resolve) => {
634
+ const proc = spawn(this._python, args, {
635
+ cwd: this._opts.cwd,
636
+ env: { ...process.env, ...this._opts.env },
637
+ stdio: ['ignore', 'pipe', 'pipe'],
638
+ });
639
+
640
+ let stdout = '';
641
+ proc.stdout.on('data', (d: Buffer) => { stdout += d.toString(); });
642
+ proc.stderr.on('data', (d: Buffer) => { stdout += d.toString(); });
643
+
644
+ const timer = setTimeout(() => {
645
+ proc.kill('SIGKILL');
646
+ stdout += '\n[TIMEOUT]';
647
+ }, this._opts.timeoutMs);
648
+
649
+ proc.on('close', (code) => {
650
+ clearTimeout(timer);
651
+ resolve({ exitCode: code ?? 1, stdout });
652
+ });
653
+ });
654
+ }
655
+
656
+ /** Merge per-package results into the aggregate InstallResult. */
657
+ private _mergeResults(
658
+ result: InstallResult,
659
+ perPackage: Array<{ spec: string; status: PackageStatus; version?: string; error?: string }>
660
+ ): void {
661
+ for (const r of perPackage) {
662
+ if (r.status === 'installed' || r.status === 'upgraded') {
663
+ result.installed.push(r.version ? `${r.spec}==${r.version}` : r.spec);
664
+ } else if (r.status === 'already_installed') {
665
+ result.alreadyInstalled.push(r.spec);
666
+ } else if (r.status === 'failed') {
667
+ result.failed.push({ spec: r.spec, error: r.error ?? 'unknown' });
668
+ }
669
+ }
670
+ }
671
+
672
+ /** Emit a typed progress event. */
673
+ private _emit(
674
+ status: PackageStatus,
675
+ pkg: string,
676
+ version?: string,
677
+ error?: string
678
+ ): void {
679
+ const ev: InstallProgressEvent = { package: pkg, status, version, error };
680
+ this.emit('progress', ev);
681
+ }
682
+
683
+ /** Parse `pip show` output into PackageMetadata. */
684
+ private _parsePipShow(raw: string): PackageMetadata {
685
+ const get = (field: string): string => {
686
+ const m = raw.match(new RegExp(`^${field}:\\s*(.*)$`, 'm'));
687
+ return m ? m[1]!.trim() : '';
688
+ };
689
+ const getList = (field: string): string[] => {
690
+ const val = get(field);
691
+ return val ? val.split(/,\s*/).filter(Boolean) : [];
692
+ };
693
+ return {
694
+ name: get('Name'),
695
+ version: get('Version'),
696
+ location: get('Location'),
697
+ summary: get('Summary'),
698
+ requires: getList('Requires'),
699
+ requiredBy: getList('Required-by'),
700
+ };
701
+ }
702
+ }
703
+
704
+ // ─── Factory helpers ────────────────────────────────────────────────────────────
705
+
706
+ /**
707
+ * Create a PackageInstaller for the system Python (or the python3 in PATH).
708
+ */
709
+ export function createSystemInstaller(
710
+ options?: Partial<Omit<PackageInstallerOptions, 'pythonExecutable'>>
711
+ ): PackageInstaller {
712
+ const python = process.env['nodepyx_PYTHON'] ?? 'python3';
713
+ return new PackageInstaller({ pythonExecutable: python, ...options });
714
+ }
715
+
716
+ /**
717
+ * Create a PackageInstaller tied to a specific virtualenv directory.
718
+ */
719
+ export function createVenvInstaller(
720
+ venvDir: string,
721
+ options?: Partial<Omit<PackageInstallerOptions, 'pythonExecutable'>>
722
+ ): PackageInstaller {
723
+ const bin = process.platform === 'win32'
724
+ ? path.join(venvDir, 'Scripts', 'python.exe')
725
+ : path.join(venvDir, 'bin', 'python');
726
+
727
+ if (!fs.existsSync(bin)) {
728
+ throw new Error(`Virtualenv python not found at: ${bin}`);
729
+ }
730
+ return new PackageInstaller({ pythonExecutable: bin, ...options });
731
+ }
732
+
733
+ /**
734
+ * Resolve the import name from a pip install spec using the built-in table.
735
+ * Exposed for testing and introspection.
736
+ */
737
+ export { resolveImportName, extractPackageName, INSTALL_TO_IMPORT };
738
+