nodepyx 1.0.0

package/src/core/PyIterator.ts

@@ -0,0 +1,294 @@
+/**
+ * nodepyx — PyIterator
+ * Wraps a Python iterator for use with JavaScript's for await...of.
+ * Provides both async iteration and manual .next() calls.
+ */
+
+import type { NativeAddon } from '../types/addon';
+import type { PyObjectId } from '../types/python';
+import { translatePythonError } from '../utils/ErrorTranslator';
+import { PyProxy, PYPROXY_INTERNAL } from './PyProxy';
+import { Logger } from '../utils/Logger';
+
+const logger = new Logger('PyIterator');
+
+export interface PyIteratorOptions {
+  /** Maximum number of items to yield (0 = no limit) */
+  limit?: number;
+  /** Whether to wrap Python objects in PyProxy */
+  wrapObjects?: boolean;
+  /** Timeout per .next() call (ms) */
+  timeout?: number;
+}
+
+/**
+ * PyIterator — manages a Python iterator lifecycle.
+ *
+ * @example
+ * ```typescript
+ * const iterator = await PyIterator.create(pyList, addon);
+ *
+ * for await (const item of iterator) {
+ *   console.log(item);
+ * }
+ *
+ * // Or manually:
+ * const { value, done } = await iterator.next();
+ * ```
+ */
+export class PyIterator<T = unknown> implements AsyncIterable<T>, AsyncIterator<T> {
+  private readonly _iteratorId: number;
+  private readonly _addon: NativeAddon;
+  private readonly _objectId: PyObjectId;
+  private readonly _options: PyIteratorOptions;
+  private _done = false;
+  private _position = 0;
+  private _destroyed = false;
+
+  private constructor(
+    iteratorId: number,
+    objectId: PyObjectId,
+    addon: NativeAddon,
+    options: PyIteratorOptions = {}
+  ) {
+    this._iteratorId = iteratorId;
+    this._objectId = objectId;
+    this._addon = addon;
+    this._options = options;
+  }
+
+  /**
+   * Create a PyIterator from a PyProxy.
+   */
+  static async create(
+    source: PyProxy,
+    options?: PyIteratorOptions
+  ): Promise<PyIterator> {
+    const internal = source[PYPROXY_INTERNAL];
+    const addon = internal.addon;
+
+    const result = await addon.createIterator(
+      internal.objectId as number,
+      internal.path
+    ) as { success?: boolean; resultJson?: string; error?: { type: string; message: string; traceback: string } };
+
+    if (!result.success) {
+      if (result.error) {
+        throw translatePythonError({
+          type: result.error.type,
+          message: result.error.message,
+          traceback: result.error.traceback || '',
+        });
+      }
+      throw new Error('Failed to create Python iterator');
+    }
+
+    let iteratorId: number;
+    try {
+      const parsed = JSON.parse(result.resultJson || '{}') as { iteratorId?: number };
+      if (!parsed.iteratorId) {throw new Error('no iteratorId');}
+      iteratorId = parsed.iteratorId;
+    } catch {
+      throw new Error('Failed to parse iterator ID from: ' + result.resultJson);
+    }
+
+    return new PyIterator(iteratorId, internal.objectId, addon, options);
+  }
+
+  /**
+   * Create a PyIterator from an object ID and iterator ID directly.
+   */
+  static fromIds(
+    iteratorId: number,
+    objectId: PyObjectId,
+    addon: NativeAddon,
+    options?: PyIteratorOptions
+  ): PyIterator {
+    return new PyIterator(iteratorId, objectId, addon, options);
+  }
+
+  /** Current position in iteration */
+  get position(): number {
+    return this._position;
+  }
+
+  /** Whether iteration is complete */
+  get isDone(): boolean {
+    return this._done;
+  }
+
+  // ─── AsyncIterator Protocol ────────────────────────────────────────────────
+
+  async next(): Promise<IteratorResult<T>> {
+    if (this._done || this._destroyed) {
+      return { value: undefined as unknown as T, done: true };
+    }
+
+    // Check limit
+    if (this._options.limit && this._position >= this._options.limit) {
+      await this.destroy();
+      return { value: undefined as unknown as T, done: true };
+    }
+
+    try {
+      const result = await this._addon.iteratorNext(this._iteratorId) as {
+        success?: boolean;
+        resultJson?: string;
+        error?: { type: string; message: string; traceback: string };
+      };
+
+      if (!result.success) {
+        if (result.error) {
+          throw translatePythonError({
+            type: result.error.type,
+            message: result.error.message,
+            traceback: result.error.traceback || '',
+          });
+        }
+        this._done = true;
+        return { value: undefined as unknown as T, done: true };
+      }
+
+      let parsed: { done?: boolean; value?: unknown };
+      try {
+        parsed = JSON.parse(result.resultJson || '{"done":true}') as { done?: boolean; value?: unknown };
+      } catch {
+        this._done = true;
+        return { value: undefined as unknown as T, done: true };
+      }
+
+      if (parsed.done) {
+        this._done = true;
+        return { value: undefined as unknown as T, done: true };
+      }
+
+      this._position++;
+
+      // Deserialize the value
+      const rawValue = parsed.value;
+      let value: T;
+
+      if (
+        rawValue !== null &&
+        typeof rawValue === 'object' &&
+        '__pyref__' in (rawValue as Record<string, unknown>)
+      ) {
+        // It's a Python object reference
+        if (this._options.wrapObjects !== false) {
+          const refId = (rawValue as { __pyref__: number }).__pyref__;
+          value = PyProxy._createFromObjectId(
+            refId as PyObjectId,
+            [],
+            this._addon
+          ) as unknown as T;
+        } else {
+          value = rawValue as T;
+        }
+      } else {
+        value = rawValue as T;
+      }
+
+      return { value, done: false };
+    } catch (err) {
+      this._done = true;
+      throw err;
+    }
+  }
+
+  async return(value?: T): Promise<IteratorResult<T>> {
+    await this.destroy();
+    return { value: value as T, done: true };
+  }
+
+  async throw(error?: unknown): Promise<IteratorResult<T>> {
+    await this.destroy();
+    throw error;
+  }
+
+  // ─── AsyncIterable Protocol ────────────────────────────────────────────────
+
+  [Symbol.asyncIterator](): AsyncIterator<T> {
+    return this;
+  }
+
+  // ─── Utilities ─────────────────────────────────────────────────────────────
+
+  /**
+   * Collect all items into an array.
+   * Use with care for large iterators!
+   */
+  async toArray(maxItems = 100000): Promise<T[]> {
+    const items: T[] = [];
+    for await (const item of this) {
+      items.push(item);
+      if (items.length >= maxItems) {break;}
+    }
+    return items;
+  }
+
+  /**
+   * Process items in batches.
+   */
+  async *batch(batchSize: number): AsyncGenerator<T[]> {
+    let current: T[] = [];
+    for await (const item of this) {
+      current.push(item);
+      if (current.length >= batchSize) {
+        yield current;
+        current = [];
+      }
+    }
+    if (current.length > 0) {
+      yield current;
+    }
+  }
+
+  /**
+   * Map items through a transform function.
+   */
+  async *map<U>(fn: (item: T, index: number) => U | Promise<U>): AsyncGenerator<U> {
+    let i = 0;
+    for await (const item of this) {
+      yield await fn(item, i++);
+    }
+  }
+
+  /**
+   * Filter items.
+   */
+  async *filter(fn: (item: T, index: number) => boolean | Promise<boolean>): AsyncGenerator<T> {
+    let i = 0;
+    for await (const item of this) {
+      if (await fn(item, i++)) {yield item;}
+    }
+  }
+
+  /**
+   * Take first N items.
+   */
+  async take(n: number): Promise<T[]> {
+    const items: T[] = [];
+    for await (const item of this) {
+      items.push(item);
+      if (items.length >= n) {break;}
+    }
+    await this.destroy();
+    return items;
+  }
+
+  /**
+   * Destroy the underlying Python iterator (frees resources).
+   * Called automatically when iteration completes.
+   */
+  async destroy(): Promise<void> {
+    if (this._destroyed) {return;}
+    this._destroyed = true;
+    this._done = true;
+    try {
+      await this._addon.destroyIterator(this._iteratorId);
+    } catch (err) {
+      logger.debug('Error destroying iterator', err);
+    }
+  }
+}
+

package/src/core/PyModule.ts

@@ -0,0 +1,194 @@
+/**
+ * nodepyx — PyModule
+ * Represents a Python module with its full namespace.
+ * Inherits from PyProxy but adds module-specific introspection.
+ */
+
+import { PyProxy, PYPROXY_INTERNAL } from './PyProxy';
+import type { NativeAddon } from '../types/addon';
+import type { PyObjectId, PyModuleInspection } from '../types/python';
+import { Logger } from '../utils/Logger';
+
+const logger = new Logger('PyModule');
+
+/**
+ * PyModule extends PyProxy with module-specific utilities.
+ *
+ * @example
+ * ```typescript
+ * const pd = await py.import('pandas');
+ * // pd is a PyModule
+ * const df = await pd.read_csv('data.csv');  // access module attribute
+ * const info = await pd.__nodepyx_inspect__();  // get module metadata
+ * ```
+ */
+export class PyModule extends PyProxy {
+  private readonly _moduleName: string;
+  private _inspection: PyModuleInspection | null = null;
+
+  constructor(
+    objectId: PyObjectId,
+    moduleName: string,
+    addon: NativeAddon,
+    callTimeout?: number
+  ) {
+    super({
+      objectId,
+      path: [],
+      addon,
+      isResolved: false,
+      callTimeout,
+      childCache: new Map(),
+    });
+    this._moduleName = moduleName;
+  }
+
+  /** The Python module name */
+  get moduleName(): string {
+    return this._moduleName;
+  }
+
+  /** Object ID in Python heap */
+  get objectId(): PyObjectId {
+    return this[PYPROXY_INTERNAL].objectId;
+  }
+
+  /**
+   * Inspect the module — returns detailed type information.
+   * Cached after first call.
+   */
+  async inspect(): Promise<PyModuleInspection> {
+    if (this._inspection) {return this._inspection;}
+
+    const addon = this[PYPROXY_INTERNAL].addon;
+
+    try {
+      // Run Python introspection
+      const result = await addon.runPythonCode(`
+import json
+import sys
+
+def _inspect_module_${this._moduleName.replace(/\./g, '_')}():
+    try:
+        import ${this._moduleName} as _mod
+        import inspect
+        
+        result = {
+            'name': '${this._moduleName}',
+            'file': getattr(_mod, '__file__', None),
+            'docstring': (getattr(_mod, '__doc__', '') or '')[:500],
+            'version': getattr(_mod, '__version__', None),
+            'functions': [],
+            'classes': [],
+            'constants': [],
+            'submodules': [],
+        }
+        
+        for name in dir(_mod):
+            if name.startswith('_'):
+                continue
+            try:
+                obj = getattr(_mod, name)
+                if inspect.ismodule(obj):
+                    result['submodules'].append(name)
+                elif inspect.isclass(obj):
+                    result['classes'].append({'name': name, 'type': 'class'})
+                elif callable(obj):
+                    result['functions'].append({'name': name, 'type': 'function'})
+                else:
+                    result['constants'].append({'name': name, 'type': type(obj).__name__})
+            except Exception:
+                pass
+        
+        return result
+    except Exception as e:
+        return {'name': '${this._moduleName}', 'error': str(e), 'functions': [], 'classes': [], 'constants': [], 'submodules': []}
+
+_result = _inspect_module_${this._moduleName.replace(/\./g, '_')}()
+json.dumps(_result)
+`);
+
+      const r = result as { success?: boolean; resultJson?: string };
+      if (r.success && r.resultJson) {
+        const parsed = JSON.parse(r.resultJson.replace(/^"|"$/g, '').replace(/\\"/g, '"'));
+        this._inspection = parsed as PyModuleInspection;
+        return this._inspection;
+      }
+    } catch (err) {
+      logger.warn(`Failed to inspect module ${this._moduleName}`, err);
+    }
+
+    // Return minimal inspection
+    this._inspection = {
+      name: this._moduleName,
+      docstring: '',
+      functions: [],
+      classes: [],
+      constants: [],
+      submodules: [],
+    };
+
+    return this._inspection;
+  }
+
+  /**
+   * Get list of public names in the module (dir() equivalent).
+   */
+  async getNames(): Promise<string[]> {
+    const addon = this[PYPROXY_INTERNAL].addon;
+    const result = await addon.getObjectKeys(this.objectId as number);
+    const r = result as { success?: boolean; resultJson?: string };
+    if (r.success && r.resultJson) {
+      try {
+        return JSON.parse(r.resultJson) as string[];
+      } catch {
+        return [];
+      }
+    }
+    return [];
+  }
+
+  /**
+   * Check if the module has a specific attribute.
+   */
+  async has(name: string): Promise<boolean> {
+    const addon = this[PYPROXY_INTERNAL].addon;
+    return addon.hasAttribute(this.objectId as number, name);
+  }
+
+  /**
+   * Get module version string.
+   */
+  async getVersion(): Promise<string | null> {
+    const inspection = await this.inspect();
+    return inspection.version ?? null;
+  }
+
+  /**
+   * Reload the module (re-imports from Python).
+   */
+  async reload(): Promise<void> {
+    const addon = this[PYPROXY_INTERNAL].addon;
+    await addon.reloadModule(this.objectId as number);
+    this._inspection = null;  // Clear inspection cache
+    this[PYPROXY_INTERNAL].childCache.clear();  // Clear attribute cache
+    logger.debug(`Module ${this._moduleName} reloaded`);
+  }
+
+  /**
+   * Create a PyModule from an existing addon import result.
+   */
+  static fromRef(
+    objectId: PyObjectId,
+    moduleName: string,
+    addon: NativeAddon,
+    callTimeout?: number
+  ): PyModule {
+    return new PyModule(objectId, moduleName, addon, callTimeout);
+  }
+
+  override toString(): string {
+    return `[PyModule '${this._moduleName}' id=${this.objectId}]`;
+  }
+}
+