nodepyx 1.0.0

package/src/native/type_converter.h

@@ -0,0 +1,272 @@
+/**
+ * nodepyx — Type Converter Header
+ * Converts between Python objects and JavaScript/JSON representations.
+ *
+ * Conversion strategy:
+ * - Primitive types (int, float, bool, str, None) → JSON
+ * - Large arrays (numpy, lists) → MessagePack binary
+ * - numpy arrays → binary buffer with metadata
+ * - pandas DataFrames → Arrow/binary with schema metadata
+ * - Complex Python objects → opaque references (PyObjectId)
+ */
+
+#pragma once
+
+#include <Python.h>
+#include <napi.h>
+#include <string>
+#include <vector>
+#include <cstdint>
+#include <optional>
+
+namespace nodepyx {
+
+/**
+ * Wire format identifiers (must match TypeScript SerializedFormat enum)
+ */
+enum class SerializedFormat : uint8_t {
+  JSON = 0,
+  MSGPACK = 1,
+  NUMPY_ARRAY = 2,
+  PANDAS_DATAFRAME = 3,
+  PANDAS_SERIES = 4,
+  TORCH_TENSOR = 5,
+  PYTHON_REF = 6,
+  BYTES = 7,
+  NONE = 8,
+};
+
+/**
+ * Metadata for serialized values.
+ */
+struct SerializedMetadata {
+  // For NUMPY_ARRAY
+  std::string dtype;
+  std::vector<int64_t> shape;
+  std::vector<int64_t> strides;
+  bool isC_contiguous = true;
+
+  // For PANDAS_DATAFRAME
+  std::vector<std::string> columns;
+  std::vector<std::string> columnDtypes;
+
+  // For PYTHON_REF
+  uint32_t objectId = 0;
+
+  // General
+  int64_t length = 0;
+  int64_t size_bytes = 0;
+};
+
+/**
+ * A serialized value ready to be sent to JavaScript.
+ */
+struct SerializedValue {
+  SerializedFormat format = SerializedFormat::NONE;
+  std::string jsonData;          // For JSON format
+  std::vector<uint8_t> binaryData; // For binary formats
+  SerializedMetadata metadata;
+};
+
+/**
+ * TypeConverter — stateless utility class for Python ↔ JavaScript conversions.
+ *
+ * All methods are static and thread-safe (GIL must be held by caller).
+ */
+class TypeConverter {
+public:
+  TypeConverter() = delete;
+
+  // ─── Python → SerializedValue ───────────────────────────────────────────
+
+  /**
+   * Convert any Python object to a SerializedValue for JavaScript consumption.
+   * @param obj - Python object (borrowed reference)
+   * @param depth - current recursion depth (prevents infinite loops)
+   * @returns SerializedValue
+   */
+  static SerializedValue pythonToSerialized(PyObject* obj, int depth = 0);
+
+  /**
+   * Convert Python primitive to JSON string.
+   * Returns std::nullopt if obj is not a primitive.
+   */
+  static std::optional<std::string> primitiveToJson(PyObject* obj);
+
+  /**
+   * Convert Python list/tuple to JSON array string.
+   */
+  static std::string listToJson(PyObject* seq, int depth);
+
+  /**
+   * Convert Python dict to JSON object string.
+   */
+  static std::string dictToJson(PyObject* mapping, int depth);
+
+  /**
+   * Serialize numpy ndarray to binary buffer + metadata.
+   */
+  static SerializedValue numpyArrayToBuffer(PyObject* array);
+
+  /**
+   * Serialize pandas DataFrame to binary + metadata.
+   */
+  static SerializedValue dataFrameToBuffer(PyObject* df);
+
+  /**
+   * Serialize pandas Series to binary + metadata.
+   */
+  static SerializedValue seriesToBuffer(PyObject* series);
+
+  /**
+   * Check if a Python object is a numpy array.
+   */
+  static bool isNumpyArray(PyObject* obj);
+
+  /**
+   * Check if a Python object is a pandas DataFrame.
+   */
+  static bool isDataFrame(PyObject* obj);
+
+  /**
+   * Check if a Python object is a pandas Series.
+   */
+  static bool isSeries(PyObject* obj);
+
+  /**
+   * Check if a Python object is a PyTorch Tensor.
+   */
+  static bool isTorchTensor(PyObject* obj);
+
+  // ─── JavaScript/JSON → Python ───────────────────────────────────────────
+
+  /**
+   * Convert a JSON string to a Python object.
+   * @returns new Python reference, or nullptr on error
+   */
+  static PyObject* jsonToPython(const std::string& json);
+
+  /**
+   * Convert a MessagePack binary to a Python object.
+   */
+  static PyObject* msgpackToPython(const std::vector<uint8_t>& data);
+
+  /**
+   * Convert a typed array buffer to a numpy array.
+   */
+  static PyObject* bufferToNumpyArray(
+    const std::vector<uint8_t>& data,
+    const std::string& dtype,
+    const std::vector<int64_t>& shape
+  );
+
+  // ─── Type Inspection ────────────────────────────────────────────────────
+
+  /**
+   * Get the Python type name of an object.
+   */
+  static std::string getTypeName(PyObject* obj);
+
+  /**
+   * Get the full qualified type name (module.class).
+   */
+  static std::string getQualifiedTypeName(PyObject* obj);
+
+  /**
+   * Check if the object is callable.
+   */
+  static bool isCallable(PyObject* obj);
+
+  /**
+   * Check if the object is iterable.
+   */
+  static bool isIterable(PyObject* obj);
+
+  /**
+   * Check if the object is async iterable.
+   */
+  static bool isAsyncIterable(PyObject* obj);
+
+  /**
+   * Check if the object is a Python module.
+   */
+  static bool isModule(PyObject* obj);
+
+  /**
+   * Check if the object is a Python class.
+   */
+  static bool isClass(PyObject* obj);
+
+  /**
+   * Get string representation (repr) of an object.
+   */
+  static std::string getRepr(PyObject* obj);
+
+  /**
+   * Get string representation (str) of an object.
+   */
+  static std::string getStr(PyObject* obj);
+
+  // ─── Error Handling ─────────────────────────────────────────────────────
+
+  /**
+   * Capture current Python exception state as strings.
+   * Clears the exception after capturing.
+   * @returns tuple of (type, message, traceback)
+   */
+  static std::tuple<std::string, std::string, std::string> captureException();
+
+  /**
+   * Convert a Python exception to JSON for transmission to JS.
+   */
+  static std::string exceptionToJson();
+
+  // ─── Object Registry ────────────────────────────────────────────────────
+  // Python objects that can't be serialized are stored in a registry
+  // and referenced by ID from JavaScript.
+
+  /**
+   * Register a Python object and return its ID.
+   * Increments reference count.
+   */
+  static uint32_t registerObject(PyObject* obj);
+
+  /**
+   * Get a Python object by ID.
+   * Returns borrowed reference (do not decref).
+   */
+  static PyObject* getObject(uint32_t id);
+
+  /**
+   * Increment reference count of a registered object.
+   */
+  static void incRef(uint32_t id);
+
+  /**
+   * Decrement reference count of a registered object.
+   * If refcount reaches 0, removes from registry.
+   */
+  static void decRef(uint32_t id);
+
+  /**
+   * Get count of registered objects (for monitoring).
+   */
+  static size_t getRegisteredCount();
+
+  /**
+   * Clear all registered objects (during shutdown).
+   */
+  static void clearRegistry();
+
+private:
+  // JSON escape a string
+  static std::string jsonEscape(const std::string& s);
+
+  // Maximum depth for recursive conversion
+  static constexpr int MAX_DEPTH = 32;
+  // Maximum list/dict size to serialize as JSON
+  static constexpr size_t MAX_INLINE_SIZE = 10000;
+};
+
+} // namespace nodepyx
+

package/src/nextjs/PyProvider.tsx

@@ -0,0 +1,123 @@
+/**
+ * nodepyx — PyProvider (React Context Provider)
+ *
+ * Wraps a Next.js / React app to provide the nodepyx runtime to the tree.
+ * Initialises nodepyx once on mount and cleans up on unmount.
+ *
+ * Server Components: nodepyx is only active on the server.
+ * Client Components: use usePython() which reads from PyContext.
+ *
+ * Usage (app/layout.tsx):
+ * ─────────────────────────────────────────────────────────────
+ *   import { PyProvider } from 'nodepyx/next';
+ *
+ *   export default function Layout({ children }) {
+ *     return (
+ *       <PyProvider config={{ virtualenv: { path: '.venv', packages: ['pandas'] } }}>
+ *         {children}
+ *       </PyProvider>
+ *     );
+ *   }
+ * ─────────────────────────────────────────────────────────────
+ */
+
+'use client';
+
+import React, {
+  createContext,
+  useContext,
+  useEffect,
+  useState,
+  useCallback,
+  type ReactNode,
+} from 'react';
+import type { nodepyxConfig } from '../types/config';
+
+// ─── Context value ─────────────────────────────────────────────────────────────
+
+export interface PyContextValue {
+  /** Whether nodepyx has finished initialising. */
+  ready:     boolean;
+  /** Error from initialisation, if any. */
+  error:     Error | null;
+  /** Call a Python function by module.function path and get the result. */
+  callPython<T = unknown>(module: string, fn: string, ...args: unknown[]): Promise<T>;
+  /** Import a Python module proxy. */
+  importModule(name: string): Promise<unknown>;
+}
+
+const PyContext = createContext<PyContextValue | null>(null);
+
+// ─── Provider ──────────────────────────────────────────────────────────────────
+
+export interface PyProviderProps {
+  children:   ReactNode;
+  config?:    nodepyxConfig;
+  /** Suppress console errors on init failure (default: false). */
+  silent?:    boolean;
+  /** Custom loading fallback while nodepyx initialises. */
+  fallback?:  ReactNode;
+}
+
+export function PyProvider({ children, config, silent = false, fallback }: PyProviderProps) {
+  const [ready, setReady] = useState(false);
+  const [error, setError] = useState<Error | null>(null);
+
+  useEffect(() => {
+    let cancelled = false;
+
+    (async () => {
+      try {
+        // Lazy-import to avoid bundling nodepyx on the client
+        const nodepyx = await import('../index');
+        await nodepyx.init(config);
+        if (!cancelled) {setReady(true);}
+      } catch (err) {
+        if (!silent) {console.error('[nodepyx] PyProvider init error:', err);}
+        if (!cancelled) {setError(err instanceof Error ? err : new Error(String(err)));}
+      }
+    })();
+
+    return () => {
+      cancelled = true;
+      import('../index').then((nodepyx) => nodepyx.shutdown()).catch(() => {/* ignore */});
+    };
+  }, []); // eslint-disable-line -- react-hooks/exhaustive-deps intentionally omitted (empty deps: run once on mount)
+
+  const callPython = useCallback(async <T = unknown>(
+    module: string,
+    fn:     string,
+    ...args: unknown[]
+  ): Promise<T> => {
+    const nodepyx = await import('../index');
+    if (!nodepyx.isInitialized()) {throw new Error('nodepyx is not initialized');}
+    const mod = await nodepyx.importModule(module) as Record<string, (...a: unknown[]) => Promise<T>>;
+    return mod[fn]!(...args);
+  }, []);
+
+  const importModule = useCallback(async (name: string): Promise<unknown> => {
+    const nodepyx = await import('../index');
+    return nodepyx.importModule(name);
+  }, []);
+
+  if (!ready && !error && fallback) {return <>{fallback}</>;}
+
+  return (
+    <PyContext.Provider value={{ ready, error, callPython, importModule }}>
+      {children}
+    </PyContext.Provider>
+  );
+}
+
+// ─── Hook ───────────────────────────────────────────────────────────────────────
+
+/**
+ * Access the PyProvider context.
+ * Must be used inside a <PyProvider> tree.
+ */
+export function usePython(): PyContextValue {
+  const ctx = useContext(PyContext);
+  if (!ctx) {throw new Error('usePython() must be used inside <PyProvider>');}
+  return ctx;
+}
+

package/src/nextjs/index.ts

@@ -0,0 +1,21 @@
+/**
+ * nodepyx/next — Next.js integration entry point.
+ *
+ * Import from 'nodepyx/next' in your Next.js project.
+ *
+ * @example
+ * // next.config.js
+ * const { withnodepyx } = require('nodepyx/next');
+ * module.exports = withnodepyx({});
+ *
+ * @example
+ * // React component
+ * import { PyProvider, usePython } from 'nodepyx/next';
+ */
+
+export { withnodepyx }               from './withnodepyx';
+export { PyProvider, usePython as usePythonContext }    from './PyProvider';
+export { usePython }                from './usePython';
+export type { PyContextValue, PyProviderProps } from './PyProvider';
+export type { PyHandle, UsePythonResult }       from './usePython';
+

package/src/nextjs/usePython.ts

@@ -0,0 +1,106 @@
+/**
+ * nodepyx — usePython hook
+ *
+ * Convenience hook for calling Python functions from React components.
+ * Provides loading/error state management, automatic cleanup, and
+ * memoised callPython helper.
+ *
+ * Usage:
+ * ─────────────────────────────────────────────────────────────
+ *   'use client';
+ *   import { usePython } from 'nodepyx/next';
+ *
+ *   export function MyComponent() {
+ *     const { data, loading, error, run } = usePython<number[]>(
+ *       async (py) => {
+ *         const np = await py.import('numpy');
+ *         const arr = await np.arange(10);
+ *         return arr.tolist();
+ *       }
+ *     );
+ *
+ *     return (
+ *       <div>
+ *         {loading && <p>Running Python...</p>}
+ *         {error   && <p>Error: {error.message}</p>}
+ *         {data    && <pre>{JSON.stringify(data)}</pre>}
+ *         <button onClick={run}>Run</button>
+ *       </div>
+ *     );
+ *   }
+ * ─────────────────────────────────────────────────────────────
+ */
+
+'use client';
+
+import { useState, useCallback, useRef } from 'react';
+
+// Minimal type for the py object exposed to the callback
+export interface PyHandle {
+  import(name: string): Promise<unknown>;
+  eval(expression: string): Promise<unknown>;
+  exec(code: string): Promise<void>;
+}
+
+export interface UsePythonResult<T> {
+  data:    T | null;
+  loading: boolean;
+  error:   Error | null;
+  /** Execute the Python callback. Returns the result or throws. */
+  run(): Promise<T>;
+  /** Reset state to initial (null, false, null). */
+  reset(): void;
+}
+
+/**
+ * React hook that wraps a Python callback with loading/error state.
+ *
+ * @param callback - Async function receiving a py handle, returns T
+ * @param deps     - Dependencies array (re-memoises callback when changed)
+ */
+export function usePython<T = unknown>(
+  callback: (py: PyHandle) => Promise<T>
+): UsePythonResult<T> {
+  const [data,    setData]    = useState<T | null>(null);
+  const [loading, setLoading] = useState(false);
+  const [error,   setError]   = useState<Error | null>(null);
+  const abortRef = useRef(false);
+
+  const run = useCallback(async (): Promise<T> => {
+    abortRef.current = false;
+    setLoading(true);
+    setError(null);
+
+    try {
+      // Lazy-import so this module is safe to import on the server
+      const nodepyx = await import('../index');
+
+      const handle: PyHandle = {
+        import: (name) => nodepyx.importModule(name),
+        eval:   (expr) => nodepyx.evalPython(expr),
+        exec:   (code) => nodepyx.execPython(code),
+      };
+
+      const result = await callback(handle);
+      if (!abortRef.current) {setData(result);}
+      return result;
+    } catch (err) {
+      const e = err instanceof Error ? err : new Error(String(err));
+      if (!abortRef.current) {setError(e);}
+      throw e;
+    } finally {
+      if (!abortRef.current) {setLoading(false);}
+    }
+  // eslint-disable-next-line -- react-hooks/exhaustive-deps intentionally omitted
+  }, [callback]);
+
+  const reset = useCallback(() => {
+    abortRef.current = true;
+    setData(null);
+    setLoading(false);
+    setError(null);
+  }, []);
+
+  return { data, loading, error, run, reset };
+}
+

package/src/nextjs/withnodepyx.ts

@@ -0,0 +1,88 @@
+/**
+ * nodepyx — withnodepyx (Next.js config wrapper)
+ *
+ * Wraps a Next.js config to:
+ *   - Exclude the native .node addon from webpack bundling
+ *   - Add node-loader for .node binary files
+ *   - Mark nodepyx as a serverExternalPackage
+ *   - Inject nodepyx_* env vars from nodepyxConfig
+ *
+ * Usage (next.config.js):
+ * ──────────────────────────────────────────────────────────────
+ *   const { withnodepyx } = require('nodepyx/next');
+ *
+ *   module.exports = withnodepyx({
+ *     // ...your Next.js config
+ *   }, {
+ *     virtualenv: { path: './.venv', packages: ['pandas', 'numpy'], autoInstall: true }
+ *   });
+ * ──────────────────────────────────────────────────────────────
+ */
+
+import type { nodepyxConfig } from '../types/config';
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+type NextConfig = Record<string, any>;
+
+export function withnodepyx(
+  nextConfig:    NextConfig  = {},
+  nodepyxConfig?: nodepyxConfig
+): NextConfig {
+  return {
+    ...nextConfig,
+
+    // Prevent webpack from bundling native addon
+    serverExternalPackages: [
+      'nodepyx',
+      ...(nextConfig.serverExternalPackages ?? []),
+    ],
+
+    webpack(webpackConfig: NextConfig, context: { isServer: boolean }) {
+      // Exclude nodepyx from client bundles
+      if (!context.isServer) {
+        webpackConfig.resolve ??= {};
+        webpackConfig.resolve.alias = { ...(webpackConfig.resolve.alias ?? {}), nodepyx: false };
+      }
+
+      // node-loader for .node binaries
+      webpackConfig.module ??= { rules: [] };
+      webpackConfig.module.rules ??= [];
+      webpackConfig.module.rules.push({ test: /\.node$/, use: [{ loader: 'node-loader' }] });
+
+      // Fallbacks
+      webpackConfig.resolve ??= {};
+      webpackConfig.resolve.fallback = { ...(webpackConfig.resolve.fallback ?? {}), 'node-gyp-build': false };
+
+      // Server-side external
+      if (context.isServer) {
+        const prev = webpackConfig.externals ?? [];
+        const nodepyxExt = (
+          { request }: { request: string },
+          cb: (err: null, result?: string) => void
+        ) => {
+          if (request === 'nodepyx' || request.startsWith('nodepyx/')) {
+            return cb(null, `commonjs ${request}`);
+          }
+          cb(null);
+        };
+        webpackConfig.externals = Array.isArray(prev) ? [...prev, nodepyxExt] : [prev, nodepyxExt];
+      }
+
+      return typeof nextConfig.webpack === 'function'
+        ? nextConfig.webpack(webpackConfig, context)
+        : webpackConfig;
+    },
+
+    env: {
+      ...nextConfig.env,
+      ...(nodepyxConfig ? {
+        nodepyx_VENV_PATH:    nodepyxConfig.virtualenv?.path              ?? '',
+        nodepyx_PACKAGES:     (nodepyxConfig.virtualenv?.packages ?? []).join(','),
+        nodepyx_AUTO_INSTALL: String(nodepyxConfig.virtualenv?.autoInstall ?? false),
+        nodepyx_THREAD_POOL:  String(nodepyxConfig.threadPoolSize          ?? 4),
+        nodepyx_LOG_LEVEL:    nodepyxConfig.logLevel                       ?? 'warn',
+      } : {}),
+    },
+  };
+}
+

package/src/plugins/Plugin.interface.ts

@@ -0,0 +1,51 @@
+/**
+ * nodepyx — Plugin Interface
+ *
+ * Every nodepyx plugin must implement this interface. Plugins can:
+ *   - Transform Python object values as they cross the JS boundary
+ *   - Register custom type mappings
+ *   - Hook into lifecycle events (init, shutdown)
+ *   - Add convenience methods to PyProxy instances
+ */
+
+import type { SerializedValue } from '../types/python';
+
+// ─── Public types ─────────────────────────────────────────────────────────────
+
+export type PythonTypeName = string;
+
+export interface PluginInitContext {
+  runtimeVersion: string;
+  pythonVersion: { major: number; minor: number; patch: number };
+  registerTypeHandler(typeName: PythonTypeName, handler: TypeTransformHandler): void;
+}
+
+export type TypeTransformHandler = (
+  raw:      SerializedValue,
+  typeHint: string,
+  metadata: Record<string, unknown>
+) => unknown | null;
+
+export interface nodepyxPlugin {
+  readonly name:             string;
+  readonly version:          string;
+  readonly supportedModules: readonly string[];
+  readonly handledTypes?:    readonly PythonTypeName[];
+  readonly description?:     string;
+  readonly homepage?:        string;
+
+  onInit?(ctx: PluginInitContext):                      Promise<void> | void;
+  onShutdown?():                                        Promise<void> | void;
+  onModuleImported?(moduleName: string, moduleOid: number): Promise<void> | void;
+
+  transformResult?: TypeTransformHandler;
+  serializeValue?(value: unknown, targetTypeName?: string): SerializedValue | null;
+}
+
+export interface PluginRegistration {
+  plugin:       nodepyxPlugin;
+  active:       boolean;
+  loadedAt:     number;
+  modulesBound: Set<string>;
+}
+