nodepyx 1.0.0

package/src/core/PyRuntime.ts

@@ -0,0 +1,504 @@
+/**
+ * nodepyx — PyRuntime
+ * The central manager for the Python interpreter lifecycle.
+ * Creates, configures, and shuts down the embedded CPython instance.
+ */
+
+import path from 'path';
+import { loadNativeAddon } from '../types/addon';
+import type { NativeAddon } from '../types/addon';
+import { resolveConfig } from '../types/config';
+import type { nodepyxConfig, ResolvednodepyxConfig } from '../types/config';
+import type { PyObjectId } from '../types/python';
+import { PyProxy } from './PyProxy';
+import { PyModule } from './PyModule';
+import { PyContext } from './PyContext';
+import { Serializer } from '../serialization/Serializer';
+import { TypeGenerator } from '../types/TypeGenerator';
+import { PluginManager } from '../plugins/PluginManager';
+import { VenvManager } from '../env/VenvManager';
+import type { VenvManagerOptions } from '../env/VenvManager';
+import type { CondaManagerOptions } from '../env/CondaManager';
+import { PythonDetector } from '../env/PythonDetector';
+import { PackageInstaller } from '../env/PackageInstaller';
+import { MemoryMonitor } from '../utils/MemoryMonitor';
+import { Logger } from '../utils/Logger';
+import {
+  translatePythonError,
+  nodepyxShutdownError,
+} from '../utils/ErrorTranslator';
+
+const logger = new Logger('PyRuntime');
+
+/**
+ * PyRuntime — manages the lifecycle of the embedded Python interpreter.
+ *
+ * @example
+ * ```typescript
+ * const runtime = await PyRuntime.create({
+ *   virtualenv: { path: '.venv', packages: ['pandas'], autoInstall: true },
+ * });
+ *
+ * const pd = await runtime.import('pandas');
+ * const df = await pd.read_csv('data.csv');
+ * const result = await df.to_dict('records');
+ *
+ * await runtime.shutdown();
+ * ```
+ */
+export class PyRuntime {
+  private readonly _config: ResolvednodepyxConfig;
+  private readonly _addon: NativeAddon;
+  private readonly _serializer: Serializer;
+  private readonly _typeGenerator: TypeGenerator;
+  private readonly _pluginManager: PluginManager;
+  private readonly _memoryMonitor: MemoryMonitor;
+  private readonly _moduleCache = new Map<string, PyModule>();
+  private _running = false;
+
+  private constructor(config: ResolvednodepyxConfig, addon: NativeAddon) {
+    this._config = config;
+    this._addon = addon;
+    this._serializer = Serializer.getInstance();
+    this._typeGenerator = new TypeGenerator({ stubsDir: config.stubsDir });
+    this._pluginManager = new PluginManager();
+    this._memoryMonitor = new MemoryMonitor({
+      gcThreshold: this._parseBytes(config.gcThreshold),
+    });
+    this._memoryMonitor.attach(addon);
+  }
+
+  /**
+   * Create and initialize a PyRuntime instance.
+   */
+  static async create(config: nodepyxConfig = {}): Promise<PyRuntime> {
+    const resolved = resolveConfig(config);
+
+    // Set log level
+    Logger.setGlobalLevel(resolved.logLevel);
+
+    logger.info('Creating PyRuntime...');
+
+    // Load the native addon
+    let addon: NativeAddon;
+    try {
+      addon = loadNativeAddon();
+      logger.debug('Native addon loaded');
+    } catch (err) {
+      logger.error('Failed to load native addon. Make sure nodepyx is built.', err);
+      throw new Error(
+        'nodepyx native addon not found. Run "npm install" or "npm run build:native".\n' +
+        `Original error: ${(err as Error).message}`
+      );
+    }
+
+    const runtime = new PyRuntime(resolved, addon);
+    await runtime._initialize(config);
+    return runtime;
+  }
+
+  // ─── Initialization ────────────────────────────────────────────────────────
+
+  private async _initialize(config: nodepyxConfig): Promise<void> {
+    const startTime = performance.now();
+
+    // 1. Detect/setup Python environment
+    const pythonExecutable = await this._setupPythonEnvironment(config);
+
+    // 2. Initialize Python interpreter via C++ addon
+    logger.debug(`Initializing Python interpreter: ${pythonExecutable}`);
+
+    await this._addon.initializePython({
+      pythonHome: this._config.pythonHome || '',
+      pythonExecutable,
+      threadPoolSize: this._config.threadPoolSize,
+      maxQueueSize: this._config.maxQueueSize,
+      gilReleaseInterval: this._config.gil.releaseInterval,
+      callTimeout: this._config.callTimeout,
+      pythonPathExtra: [
+        path.join(__dirname, '..', '..', 'python'),
+        ...this._config.pythonPathExtra,
+      ],
+      env: this._config.env,
+      enableProfiling: this._config.profilingEnabled,
+    });
+
+    logger.debug('Python interpreter initialized');
+
+    // 3. Bootstrap Python runtime helpers
+    await this._bootstrapPythonRuntime();
+
+    // 4. Register built-in plugins
+    await this._pluginManager.registerBuiltins();
+
+    // 5. Start memory monitoring
+    this._memoryMonitor.startMonitoring(10000);
+
+    this._running = true;
+
+    const elapsed = (performance.now() - startTime).toFixed(0);
+    logger.success(`PyRuntime ready in ${elapsed}ms (Python: ${pythonExecutable})`);
+  }
+
+  private async _setupPythonEnvironment(config: nodepyxConfig): Promise<string> {
+    let pythonExecutable = this._config.pythonExecutable;
+
+    if (config.conda) {
+      // Conda environment
+      const { CondaManager } = await import('../env/CondaManager');
+      const condaManager = new CondaManager({ conda: config.conda } as CondaManagerOptions);
+      pythonExecutable = await condaManager.getPythonExecutable(config.conda.envName) ?? pythonExecutable;
+      logger.info(`Using conda environment: ${config.conda.envName}`);
+    } else if (config.virtualenv) {
+      // Virtualenv
+      const venvManager = new VenvManager({ virtualenv: config.virtualenv } as VenvManagerOptions);
+      const venvResult = await venvManager.setup();
+      pythonExecutable = typeof venvResult === 'string' ? venvResult : (venvResult as { pythonExecutable: string }).pythonExecutable ?? pythonExecutable;
+      logger.info(`Using virtualenv: ${config.virtualenv.path}`);
+    } else if (config.pythonPath) {
+      // Explicit Python path
+      pythonExecutable = config.pythonPath;
+    } else {
+      // Auto-detect
+      const detector = new PythonDetector();
+      try {
+        const detected = await detector.detect();
+        pythonExecutable = detected.executable;
+        logger.info(`Auto-detected Python: ${detected.executable} (${detected.version})`);
+      } catch {
+        logger.warn('No Python detected — using system default "python3"');
+        pythonExecutable = 'python3';
+      }
+    }
+
+    return pythonExecutable;
+  }
+
+  private async _bootstrapPythonRuntime(): Promise<void> {
+    try {
+      // Initialize nodepyx Python runtime helpers
+      const bootstrapCode = `
+import sys
+import os
+
+# Add nodepyx runtime to path
+_nodepyx_runtime_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'python')
+if _nodepyx_runtime_dir not in sys.path:
+    sys.path.insert(0, _nodepyx_runtime_dir)
+
+# Initialize runtime
+try:
+    import nodepyx_runtime
+    nodepyx_runtime.initialize()
+except ImportError:
+    pass  # Runtime helpers not available, basic operation only
+`;
+
+      await this._addon.runPythonCode(bootstrapCode);
+      logger.debug('Python runtime helpers bootstrapped');
+    } catch (err) {
+      // Non-fatal — runtime helpers are optional
+      logger.warn('Failed to bootstrap Python runtime helpers', err);
+    }
+  }
+
+  // ─── Module Import ──────────────────────────────────────────────────────────
+
+  /**
+   * Import a Python module and return a PyProxy.
+   *
+   * @example
+   * ```typescript
+   * const pd = await runtime.import('pandas');
+   * const df = await pd.read_csv('data.csv');
+   * ```
+   */
+  async import(moduleName: string): Promise<PyModule> {
+    this._ensureRunning();
+
+    // Check cache
+    if (this._moduleCache.has(moduleName)) {
+      return this._moduleCache.get(moduleName)!;
+    }
+
+    logger.debug(`Importing Python module: ${moduleName}`);
+
+    const result = await this._addon.importModule(moduleName) as {
+      success?: boolean;
+      objectId?: number;
+      isObject?: boolean;
+      error?: { type: string; message: string; traceback: string };
+    };
+
+    if (!result.success) {
+      if (result.error) {
+        throw translatePythonError({
+          type: result.error.type,
+          message: result.error.message,
+          traceback: result.error.traceback || '',
+        });
+      }
+      throw new Error(`Failed to import Python module '${moduleName}'`);
+    }
+
+    if (!result.objectId) {
+      throw new Error(`Module '${moduleName}' imported but no object ID returned`);
+    }
+
+    // Run plugin hooks
+    await this._pluginManager.onModuleImported(moduleName, result.objectId as number);
+
+    // Create PyModule
+    const module = PyModule.fromRef(
+      result.objectId as PyObjectId,
+      moduleName,
+      this._addon,
+      this._config.callTimeout
+    );
+
+    // Generate TypeScript stubs (background, non-blocking)
+    // TypeScript stub generation (non-blocking, background)
+    if (this._config.proxyCache) {
+      this._typeGenerator.generateStub(moduleName, { name: moduleName, functions: [], classes: [], constants: [], submodules: [] }).catch((err: unknown) => {
+        logger.debug(`Stub generation for ${moduleName} failed (non-fatal)`, err);
+      });
+    }
+
+    // Cache
+    this._moduleCache.set(moduleName, module);
+
+    logger.debug(`Module '${moduleName}' imported (id=${result.objectId})`);
+    return module;
+  }
+
+  /**
+   * Import multiple modules at once (concurrent).
+   */
+  async importAll(moduleNames: string[]): Promise<Record<string, PyModule>> {
+    const modules = await Promise.all(
+      moduleNames.map(async name => [name, await this.import(name)] as const)
+    );
+    return Object.fromEntries(modules);
+  }
+
+  // ─── Code Execution ────────────────────────────────────────────────────────
+
+  /**
+   * Evaluate a Python expression.
+   */
+  async eval(expression: string): Promise<unknown> {
+    this._ensureRunning();
+
+    const result = await this._addon.evalPython(expression) as {
+      success?: boolean;
+      resultJson?: string;
+      objectId?: number;
+      isObject?: boolean;
+      error?: { type: string; message: string; traceback: string };
+    };
+
+    if (!result.success) {
+      if (result.error) {
+        throw translatePythonError({
+          type: result.error.type,
+          message: result.error.message,
+          traceback: result.error.traceback || '',
+        });
+      }
+      throw new Error('Python eval failed');
+    }
+
+    if (result.isObject && result.objectId) {
+      return PyProxy._createFromObjectId(
+        result.objectId as PyObjectId,
+        [],
+        this._addon,
+        this._config.callTimeout
+      );
+    }
+
+    if (result.resultJson !== undefined) {
+      try {
+        return JSON.parse(result.resultJson);
+      } catch {
+        return result.resultJson;
+      }
+    }
+
+    return null;
+  }
+
+  /**
+   * Execute Python statements.
+   */
+  async exec(code: string): Promise<void> {
+    this._ensureRunning();
+
+    const result = await this._addon.runPythonCode(code) as {
+      success?: boolean;
+      error?: { type: string; message: string; traceback: string };
+    };
+
+    if (!result.success && result.error) {
+      throw translatePythonError({
+        type: result.error.type,
+        message: result.error.message,
+        traceback: result.error.traceback || '',
+      });
+    }
+  }
+
+  /**
+   * Run a Python script file.
+   */
+  async runFile(filePath: string): Promise<unknown> {
+    this._ensureRunning();
+
+    const result = await this._addon.runPythonFile(filePath) as {
+      success?: boolean;
+      resultJson?: string;
+      objectId?: number;
+      isObject?: boolean;
+      error?: { type: string; message: string; traceback: string };
+    };
+
+    if (!result.success) {
+      if (result.error) {
+        throw translatePythonError({
+          type: result.error.type,
+          message: result.error.message,
+          traceback: result.error.traceback || '',
+        });
+      }
+      throw new Error('Failed to run Python file');
+    }
+
+    return null;
+  }
+
+  // ─── Context Management ─────────────────────────────────────────────────────
+
+  /**
+   * Create an isolated Python execution context.
+   */
+  async createContext(): Promise<PyContext> {
+    this._ensureRunning();
+    return PyContext.create(this._addon);
+  }
+
+  // ─── Package Management ────────────────────────────────────────────────────
+
+  /**
+   * Install Python packages into the active environment.
+   */
+  async installPackages(packages: string[]): Promise<void> {
+    this._ensureRunning();
+
+    const installer = new PackageInstaller({ pythonExecutable: this._config.pythonExecutable });
+    await installer.install(packages);
+
+    // Clear module cache so re-imports work
+    this._moduleCache.clear();
+  }
+
+  // ─── Memory Management ─────────────────────────────────────────────────────
+
+  /**
+   * Get current memory statistics.
+   */
+  getMemoryStats(): ReturnType<NativeAddon['getMemoryStats']> {
+    this._ensureRunning();
+    return this._addon.getMemoryStats();
+  }
+
+  /**
+   * Trigger Python garbage collection.
+   */
+  async collectGarbage(): Promise<void> {
+    this._ensureRunning();
+    await this._addon.collectGarbage();
+  }
+
+  // ─── Direct Addon Access ───────────────────────────────────────────────────
+
+  /**
+   * Get the underlying native addon.
+   * Advanced use only.
+   */
+  getAddon(): NativeAddon {
+    return this._addon;
+  }
+
+  // ─── State ─────────────────────────────────────────────────────────────────
+
+  isRunning(): boolean {
+    return this._running;
+  }
+
+  getConfig(): ResolvednodepyxConfig {
+    return { ...this._config };
+  }
+
+  // ─── Shutdown ──────────────────────────────────────────────────────────────
+
+  /**
+   * Gracefully shut down the Python runtime.
+   * Releases all Python objects and finalizes the interpreter.
+   */
+  async shutdown(): Promise<void> {
+    if (!this._running) {return;}
+
+    logger.info('Shutting down PyRuntime...');
+    this._running = false;
+
+    // Stop memory monitoring
+    this._memoryMonitor.stopMonitoring();
+
+    // Clear caches
+    this._moduleCache.clear();
+
+    // Finalize Python interpreter
+    try {
+      await this._addon.finalizePython();
+      logger.success('PyRuntime shut down');
+    } catch (err) {
+      logger.error('Error during Python finalization', err);
+      throw err;
+    }
+  }
+
+  /**
+   * Synchronous shutdown (for process.on('exit') handler).
+   * Best-effort only — may not complete all cleanup.
+   */
+  shutdownSync(): void {
+    if (!this._running) {return;}
+    this._running = false;
+    this._memoryMonitor.stopMonitoring();
+    this._moduleCache.clear();
+    // Cannot await here — best effort
+    logger.debug('Synchronous shutdown initiated');
+  }
+
+  // ─── Private Helpers ───────────────────────────────────────────────────────
+
+  private _ensureRunning(): void {
+    if (!this._running) {
+      throw new nodepyxShutdownError(
+        'PyRuntime has been shut down. Create a new runtime with PyRuntime.create().'
+      );
+    }
+  }
+
+  private _parseBytes(str: string): number {
+    const match = str.match(/^(\d+(?:\.\d+)?)\s*(B|KB|MB|GB|TB)?$/i);
+    if (!match) {return 1_073_741_824;}
+    const value = parseFloat(match[1]);
+    const unit = (match[2] ?? 'B').toUpperCase();
+    const units: Record<string, number> = {
+      B: 1, KB: 1024, MB: 1024 ** 2, GB: 1024 ** 3, TB: 1024 ** 4,
+    };
+    return Math.floor(value * (units[unit] ?? 1));
+  }
+}
+