nodepyx 1.0.0

package/src/types/TypeGenerator.ts

@@ -0,0 +1,491 @@
+/**
+ * nodepyx — TypeGenerator
+ * Generates TypeScript .d.ts stub files from Python module introspection.
+ *
+ * Flow:
+ *   1. Check StubCache — return cached stub if fresh.
+ *   2. Run Python introspection via addon.runPythonCode()
+ *      (executes type_inspector.py in the embedded Python runtime).
+ *   3. Parse the JSON introspection result.
+ *   4. Generate .d.ts content using PythonTypeMapper.
+ *   5. Store in StubCache.
+ *   6. Return type info array (PyTypeInfo[]).
+ *
+ * Generated stubs are compatible with TypeScript LSP — they provide
+ * real autocomplete inside editors without running Python.
+ */
+
+import * as path from 'path';
+import { PythonTypeMapper, mapReturnType } from './PythonTypeMapper';
+import { StubCache } from './StubCache';
+import { Logger } from '../utils/Logger';
+import type {
+  PyModuleInspection,
+  PyCallableInfo,
+  PyClassInfo,
+  PyMemberInfo,
+  PyParamInfo,
+} from './python';
+
+const logger = new Logger('TypeGenerator');
+
+// ─── Re-export for convenience ────────────────────────────────────────────
+
+export type { PyModuleInspection, PyCallableInfo, PyClassInfo, PyMemberInfo };
+
+// ─── Options ─────────────────────────────────────────────────────────────
+
+export interface TypeGeneratorOptions {
+  /** Directory to store generated .d.ts files. Default: './.nodepyx/stubs' */
+  stubsDir?: string;
+  /** nodepyx version string for cache validation */
+  nodepyxVersion?: string;
+  /**
+   * Whether to emit strict types (never fall back to 'unknown').
+   * Default: false
+   */
+  strictTypes?: boolean;
+  /**
+   * Maximum number of members to include per class in the generated stub.
+   * Default: 200
+   */
+  maxMembersPerClass?: number;
+  /**
+   * Whether to include private members (names starting with _).
+   * Default: false
+   */
+  includePrivate?: boolean;
+  /**
+   * Whether to include inherited members.
+   * Default: false
+   */
+  includeInherited?: boolean;
+}
+
+// ─── Raw introspection JSON schema (from type_inspector.py) ───────────────
+
+export interface RawParamInfo {
+  name: string;
+  type?: string;
+  annotation?: string;
+  optional?: boolean;
+  has_default?: boolean;
+  default?: string;
+  kind?: string;
+}
+
+export interface RawCallableInfo {
+  name: string;
+  qual_name?: string;
+  docstring?: string;
+  parameters?: RawParamInfo[];
+  return_type?: string;
+  return_annotation?: string;
+  is_method?: boolean;
+  is_class_method?: boolean;
+  is_static_method?: boolean;
+  is_coroutine?: boolean;
+  is_generator?: boolean;
+}
+
+export interface RawMemberInfo {
+  name: string;
+  type?: string;
+  read_only?: boolean;
+  docstring?: string;
+}
+
+export interface RawClassInfo {
+  name: string;
+  qual_name?: string;
+  docstring?: string;
+  bases?: string[];
+  members?: RawMemberInfo[];
+  methods?: RawCallableInfo[];
+  properties?: RawMemberInfo[];
+  class_methods?: RawCallableInfo[];
+  static_methods?: RawCallableInfo[];
+}
+
+export interface RawModuleInspection {
+  name: string;
+  file?: string;
+  docstring?: string;
+  version?: string;
+  functions?: RawCallableInfo[];
+  classes?: RawClassInfo[];
+  constants?: RawMemberInfo[];
+  submodules?: string[];
+  error?: string;
+}
+
+/**
+ * TypeGenerator — generates TypeScript type stubs from Python modules.
+ *
+ * @example
+ * ```typescript
+ * const gen = new TypeGenerator({ stubsDir: './.nodepyx/stubs' });
+ *
+ * // Generate stub (uses cache if available)
+ * const dts = await gen.generateStub('pandas');
+ *
+ * // Access parsed type info
+ * const info = await gen.inspectModule('pandas');
+ * console.log(info.classes.map(c => c.name));
+ * ```
+ */
+export class TypeGenerator {
+  private readonly _options: Required<TypeGeneratorOptions>;
+  private readonly _cache: StubCache;
+
+  constructor(options: TypeGeneratorOptions = {}) {
+    this._options = {
+      stubsDir:          options.stubsDir          ?? path.join(process.cwd(), '.nodepyx', 'stubs'),
+      nodepyxVersion:     options.nodepyxVersion     ?? '1.0.0',
+      strictTypes:       options.strictTypes        ?? false,
+      maxMembersPerClass: options.maxMembersPerClass ?? 200,
+      includePrivate:    options.includePrivate     ?? false,
+      includeInherited:  options.includeInherited   ?? false,
+    };
+
+    this._cache = new StubCache({
+      stubsDir:      this._options.stubsDir,
+      nodepyxVersion: this._options.nodepyxVersion,
+    });
+
+    // Pre-populate cache with bundled stubs for common libraries
+    this._cache.initPrebuilts();
+  }
+
+  // ─── Public API ──────────────────────────────────────────────────────────
+
+  /**
+   * Generate (or return cached) TypeScript stub for a Python module.
+   * Returns the .d.ts content as a string.
+   */
+  async generateStub(
+    moduleName: string,
+    inspection: RawModuleInspection,
+    libVersion?: string,
+  ): Promise<string> {
+    // ── Cache check ──────────────────────────────────────────────────────
+    const cached = this._cache.get(moduleName, libVersion);
+    if (cached) {
+      logger.debug(`TypeGenerator: cache hit for '${moduleName}'`);
+      return cached;
+    }
+
+    logger.info(`TypeGenerator: generating stub for '${moduleName}'`);
+
+    if (inspection.error) {
+      logger.warn(`TypeGenerator: introspection error for '${moduleName}': ${inspection.error}`);
+      const fallback = this._generateFallbackStub(moduleName, inspection.error);
+      this._cache.set(moduleName, fallback, { libVersion });
+      return fallback;
+    }
+
+    const dts = this._generateDTS(moduleName, inspection);
+    this._cache.set(moduleName, dts, { libVersion });
+
+    return dts;
+  }
+
+  /**
+   * Parse raw introspection JSON from Python into typed structures.
+   */
+  parseInspection(raw: RawModuleInspection): PyModuleInspection {
+    return {
+      name:       raw.name,
+      file:       raw.file,
+      docstring:  raw.docstring ?? '',
+      version:    raw.version,
+      functions:  (raw.functions ?? []).map(f => this._parseCallable(f)),
+      classes:    (raw.classes   ?? []).map(c => this._parseClass(c)),
+      constants:  (raw.constants ?? []).map(m => this._parseMember(m)),
+      submodules: raw.submodules ?? [],
+    };
+  }
+
+  /**
+   * Return the StubCache instance.
+   */
+  get cache(): StubCache {
+    return this._cache;
+  }
+
+  // ─── DTS Generation ───────────────────────────────────────────────────────
+
+  private _generateDTS(moduleName: string, raw: RawModuleInspection): string {
+    const mapper = new PythonTypeMapper({
+      moduleName,
+      strict: this._options.strictTypes,
+      knownClasses: new Set((raw.classes ?? []).map(c => c.name)),
+    });
+
+    const lines: string[] = [];
+
+    // ── File header ──────────────────────────────────────────────────────
+    lines.push(`// =============================================================`);
+    lines.push(`// nodepyx — Auto-generated TypeScript stub`);
+    lines.push(`// Module : ${moduleName}`);
+    if (raw.version) {lines.push(`// Version: ${raw.version}`);}
+    lines.push(`// Generated: ${new Date().toISOString()}`);
+    lines.push(`// DO NOT EDIT — regenerate with: npx nodepyx generate-stubs ${moduleName}`);
+    lines.push(`// =============================================================`);
+    lines.push(``);
+
+    // ── Shared type imports ───────────────────────────────────────────────
+    lines.push(`import type { PyProxy, DataFrameResult, SeriesResult } from 'nodepyx';`);
+    lines.push(``);
+
+    // ── Module declaration ────────────────────────────────────────────────
+    lines.push(`declare module 'nodepyx/${moduleName}' {`);
+    lines.push(``);
+
+    // Module docstring
+    if (raw.docstring) {
+      lines.push(...this._docComment(raw.docstring, 2));
+    }
+
+    // ── Constants ────────────────────────────────────────────────────────
+    for (const constant of raw.constants ?? []) {
+      if (!this._options.includePrivate && constant.name.startsWith('_')) {continue;}
+      if (constant.docstring) {lines.push(...this._docComment(constant.docstring, 2));}
+      const tsType = mapper.map(constant.type ?? 'Any');
+      lines.push(`  export const ${constant.name}: ${tsType};`);
+    }
+
+    if ((raw.constants ?? []).length > 0) {lines.push(``);}
+
+    // ── Top-level functions ───────────────────────────────────────────────
+    for (const fn of raw.functions ?? []) {
+      if (!this._options.includePrivate && fn.name.startsWith('_')) {continue;}
+      lines.push(...this._functionDecl(fn, mapper, 2, 'export function'));
+      lines.push(``);
+    }
+
+    // ── Classes ───────────────────────────────────────────────────────────
+    for (const cls of raw.classes ?? []) {
+      if (!this._options.includePrivate && cls.name.startsWith('_')) {continue;}
+      lines.push(...this._classDecl(cls, mapper));
+      lines.push(``);
+    }
+
+    // ── Sub-module re-exports ─────────────────────────────────────────────
+    for (const sub of raw.submodules ?? []) {
+      lines.push(`  export const ${sub}: PyProxy;`);
+    }
+
+    lines.push(`}`);
+    lines.push(``);
+
+    return lines.join('\n');
+  }
+
+  private _functionDecl(
+    fn: RawCallableInfo,
+    mapper: PythonTypeMapper,
+    indent: number,
+    prefix: string,
+  ): string[] {
+    const pad = ' '.repeat(indent);
+    const lines: string[] = [];
+
+    if (fn.docstring) {
+      lines.push(...this._docComment(fn.docstring, indent));
+    }
+
+    const params = (fn.parameters ?? [])
+      .filter(p => p.name !== 'self' && p.name !== 'cls')
+      .filter(p => this._options.includePrivate || !p.name.startsWith('_'))
+      .slice(0, 30)  // sanity cap
+      .map(p => this._paramDecl(p, mapper));
+
+    const retType = mapReturnType(
+      fn.return_type ?? fn.return_annotation ?? 'None',
+      { strict: this._options.strictTypes },
+    );
+
+    lines.push(`${pad}${prefix} ${fn.name}(${params.join(', ')}): ${retType};`);
+
+    return lines;
+  }
+
+  private _paramDecl(p: RawParamInfo, mapper: PythonTypeMapper): string {
+    const name = p.name;
+    const raw  = p.type ?? p.annotation ?? 'Any';
+    const isOptional = p.optional ?? p.has_default ?? false;
+    const isVarArgs  = p.kind === 'var_positional';
+    const isKwArgs   = p.kind === 'var_keyword';
+
+    const tsType = mapper.map(raw);
+
+    if (isVarArgs) {
+      return `...${name}: Array<${tsType}>`;
+    }
+    if (isKwArgs) {
+      return `${name}?: Record<string, ${tsType}>`;
+    }
+    if (isOptional) {
+      return `${name}?: ${tsType} | null`;
+    }
+    return `${name}: ${tsType}`;
+  }
+
+  private _classDecl(cls: RawClassInfo, mapper: PythonTypeMapper): string[] {
+    const lines: string[] = [];
+    const pad = '  ';
+
+    if (cls.docstring) {
+      lines.push(...this._docComment(cls.docstring, 2));
+    }
+
+    const bases = (cls.bases ?? [])
+      .filter(b => b !== 'object')
+      .map(b => mapper.map(b))
+      .join(', ');
+
+    lines.push(`${pad}export class ${cls.name}${bases ? ` extends ${bases}` : ''} {`);
+
+    // Constructor
+    const initMethod = (cls.methods ?? []).find(m => m.name === '__init__');
+    if (initMethod) {
+      const ctorParams = (initMethod.parameters ?? [])
+        .filter(p => p.name !== 'self')
+        .map(p => this._paramDecl(p, mapper));
+      lines.push(`${pad}  constructor(${ctorParams.join(', ')});`);
+    } else {
+      lines.push(`${pad}  constructor(...args: unknown[]);`);
+    }
+
+    // Properties
+    const allProps = [
+      ...(cls.properties ?? []),
+      ...(cls.members    ?? []).filter(m => !m.name.startsWith('__')),
+    ].slice(0, this._options.maxMembersPerClass);
+
+    for (const prop of allProps) {
+      if (!this._options.includePrivate && prop.name.startsWith('_')) {continue;}
+      const tsType = mapper.map(prop.type ?? 'Any');
+      const ro = prop.read_only ? 'readonly ' : '';
+      lines.push(`${pad}  ${ro}${prop.name}: Promise<${tsType}>;`);
+    }
+
+    // Methods
+    const allMethods = [
+      ...(cls.methods        ?? []).filter(m => m.name !== '__init__'),
+      ...(cls.static_methods ?? []),
+      ...(cls.class_methods  ?? []),
+    ].slice(0, this._options.maxMembersPerClass);
+
+    for (const method of allMethods) {
+      if (!this._options.includePrivate && method.name.startsWith('_')) {
+        if (!['__len__', '__str__', '__repr__', '__iter__'].includes(method.name)) {continue;}
+      }
+
+      if (method.docstring) {
+        lines.push(...this._docComment(method.docstring, 4));
+      }
+
+      const isStatic = (cls.static_methods ?? []).some(s => s.name === method.name);
+      const prefix   = isStatic ? 'static ' : '';
+
+      const params = (method.parameters ?? [])
+        .filter(p => p.name !== 'self' && p.name !== 'cls')
+        .slice(0, 20)
+        .map(p => this._paramDecl(p, mapper));
+
+      const retType = mapReturnType(
+        method.return_type ?? method.return_annotation ?? 'None',
+        { strict: this._options.strictTypes },
+      );
+
+      lines.push(`${pad}  ${prefix}${method.name}(${params.join(', ')}): ${retType};`);
+    }
+
+    lines.push(`${pad}  [key: string]: unknown;`);
+    lines.push(`${pad}}`);
+
+    return lines;
+  }
+
+  private _generateFallbackStub(moduleName: string, error: string): string {
+    return [
+      `// nodepyx — Fallback stub for '${moduleName}'`,
+      `// Introspection error: ${error}`,
+      `// Install the module to get proper types.`,
+      ``,
+      `declare module 'nodepyx/${moduleName}' {`,
+      `  export const [key: string]: unknown;`,
+      `}`,
+      ``,
+    ].join('\n');
+  }
+
+  // ─── Parsing helpers ──────────────────────────────────────────────────────
+
+  private _parseCallable(raw: RawCallableInfo): PyCallableInfo {
+    return {
+      name:           raw.name,
+      qualName:       raw.qual_name ?? raw.name,
+      docstring:      raw.docstring ?? '',
+      parameters:     (raw.parameters ?? []).map(p => this._parseParam(p)),
+      returnType:     raw.return_type ?? raw.return_annotation ?? 'None',
+      isMethod:       raw.is_method       ?? false,
+      isClassMethod:  raw.is_class_method  ?? false,
+      isStaticMethod: raw.is_static_method ?? false,
+      isCoroutine:    raw.is_coroutine     ?? false,
+      isGenerator:    raw.is_generator     ?? false,
+    };
+  }
+
+  private _parseParam(raw: RawParamInfo): PyParamInfo {
+    return {
+      name:         raw.name,
+      type:         raw.type ?? raw.annotation ?? 'Any',
+      optional:     raw.optional ?? raw.has_default ?? false,
+      hasDefault:   raw.has_default ?? false,
+      defaultValue: raw.default,
+      kind:         (raw.kind as PyParamInfo['kind']) ?? 'positional_or_keyword',
+    };
+  }
+
+  private _parseClass(raw: RawClassInfo): PyClassInfo {
+    return {
+      name:          raw.name,
+      qualName:      raw.qual_name ?? raw.name,
+      docstring:     raw.docstring ?? '',
+      bases:         raw.bases     ?? [],
+      members:       (raw.members        ?? []).map(m => this._parseMember(m)),
+      methods:       (raw.methods        ?? []).map(f => this._parseCallable(f)),
+      properties:    (raw.properties     ?? []).map(m => this._parseMember(m)),
+      classMethods:  (raw.class_methods  ?? []).map(f => this._parseCallable(f)),
+      staticMethods: (raw.static_methods ?? []).map(f => this._parseCallable(f)),
+    };
+  }
+
+  private _parseMember(raw: RawMemberInfo): PyMemberInfo {
+    return {
+      name:       raw.name,
+      type:       raw.type ?? 'Any',
+      isReadOnly: raw.read_only ?? false,
+      docstring:  raw.docstring,
+    };
+  }
+
+  // ─── JSDoc helpers ─────────────────────────────────────────────────────────
+
+  private _docComment(docstring: string, indent: number): string[] {
+    const pad   = ' '.repeat(indent);
+    const lines = docstring.split('\n').slice(0, 8); // cap at 8 lines
+    if (lines.length === 1) {
+      return [`${pad}/** ${lines[0]!.trim()} */`];
+    }
+    return [
+      `${pad}/**`,
+      ...lines.map(l => `${pad} * ${l.trim()}`),
+      `${pad} */`,
+    ];
+  }
+}
+

package/src/types/addon.ts

@@ -0,0 +1,170 @@
+/**
+ * nodepyx — Native Addon TypeScript Interface
+ * Type declarations for the C++ N-API addon.
+ * These map exactly to the exported functions in nodepyx_addon.cpp.
+ */
+
+import type {
+  SerializedValue,
+  PyCallResult,
+  PyIteratorNext,
+  PyTypeInfo,
+  AttributePath,
+} from './python';
+
+export interface AddonInitOptions {
+  /** Path to Python home directory */
+  pythonHome: string;
+  /** Path to Python executable */
+  pythonExecutable: string;
+  /** Thread pool size */
+  threadPoolSize: number;
+  /** Max thread queue size */
+  maxQueueSize: number;
+  /** GIL release interval (microseconds) */
+  gilReleaseInterval: number;
+  /** Call timeout (milliseconds) */
+  callTimeout: number;
+  /** Additional Python sys.path entries */
+  pythonPathExtra: string[];
+  /** Environment variables */
+  env: Record<string, string>;
+  /** Enable profiling */
+  enableProfiling: boolean;
+}
+
+export interface AddonImportResult {
+  objectId: number;
+  success: boolean;
+  error?: string;
+}
+
+export interface AddonAttributeResult {
+  success: boolean;
+  value?: SerializedValue;
+  objectId?: number;
+  isCallable: boolean;
+  isObject: boolean;
+  error?: string;
+}
+
+export interface AddonCallOptions {
+  /** Positional arguments */
+  args: SerializedValue[];
+  /** Keyword arguments */
+  kwargs: Record<string, SerializedValue>;
+  /** Call timeout override (ms) */
+  timeout?: number;
+}
+
+export interface AddonIteratorResult {
+  iteratorId: number;
+  success: boolean;
+  error?: string;
+}
+
+export interface AddonMemoryStats {
+  pythonHeapBytes: number;
+  pythonHeapPeakBytes: number;
+  trackedObjects: number;
+  pendingCallbacks: number;
+  threadPoolActive: number;
+  threadPoolQueued: number;
+}
+
+export interface AddonProfilingReport {
+  totalCalls: number;
+  totalTimeMs: number;
+  avgCallTimeMs: number;
+  peakCallTimeMs: number;
+  callsByModule: Record<string, number>;
+  callsByFunction: Record<string, number>;
+}
+
+/**
+ * The full interface of the compiled C++ addon module.
+ * All async methods return Promises (resolved on libuv callback).
+ */
+export interface NativeAddon {
+  // ─── Lifecycle ────────────────────────────────────────────────────────────
+  initializePython(options: AddonInitOptions): Promise<void>;
+  finalizePython(): Promise<void>;
+  isInitialized(): boolean;
+
+  // ─── Module Import ─────────────────────────────────────────────────────────
+  importModule(moduleName: string): Promise<AddonImportResult>;
+  reloadModule(objectId: number): Promise<AddonImportResult>;
+
+  // ─── Attribute Access ──────────────────────────────────────────────────────
+  getAttribute(objectId: number, name: string): Promise<AddonAttributeResult>;
+  getAttributePath(objectId: number, path: AttributePath): Promise<AddonAttributeResult>;
+  setAttribute(objectId: number, name: string, value: SerializedValue): Promise<boolean>;
+  hasAttribute(objectId: number, name: string): Promise<boolean>;
+  deleteAttribute(objectId: number, name: string): Promise<boolean>;
+
+  // ─── Function Calls ────────────────────────────────────────────────────────
+  callFunction(
+    objectId: number,
+    path: AttributePath,
+    options: AddonCallOptions
+  ): Promise<PyCallResult>;
+  callMethod(
+    objectId: number,
+    methodName: string,
+    options: AddonCallOptions
+  ): Promise<PyCallResult>;
+
+  // ─── Object Inspection ─────────────────────────────────────────────────────
+  getTypeInfo(objectId: number): Promise<PyTypeInfo>;
+  getObjectValue(objectId: number): Promise<SerializedValue>;
+  getObjectRepr(objectId: number): Promise<string>;
+  getObjectLength(objectId: number): Promise<number>;
+  getObjectKeys(objectId: number): Promise<string[]>;
+
+  // ─── Iterator Support ──────────────────────────────────────────────────────
+  createIterator(objectId: number, path: AttributePath): Promise<AddonIteratorResult>;
+  iteratorNext(iteratorId: number): Promise<PyIteratorNext>;
+  destroyIterator(iteratorId: number): Promise<void>;
+
+  // ─── Raw Python Execution ──────────────────────────────────────────────────
+  runPythonCode(code: string, globals?: SerializedValue): Promise<SerializedValue>;
+  runPythonFile(filePath: string): Promise<SerializedValue>;
+  evalPython(expression: string): Promise<SerializedValue>;
+
+  // ─── Object Lifecycle ──────────────────────────────────────────────────────
+  incRef(objectId: number): void;
+  decRef(objectId: number): void;
+  collectGarbage(): Promise<void>;
+
+  // ─── Memory & Diagnostics ─────────────────────────────────────────────────
+  getMemoryStats(): AddonMemoryStats;
+  getProfilingReport(): AddonProfilingReport;
+  resetProfilingStats(): void;
+
+  // ─── Threading ─────────────────────────────────────────────────────────────
+  setThreadPoolSize(size: number): Promise<void>;
+  getThreadPoolStats(): { active: number; queued: number; total: number };
+}
+
+/**
+ * Loads the native addon, trying prebuilds first then build directory.
+ */
+export function loadNativeAddon(): NativeAddon {
+  // Try prebuilt binary first (faster, no build tools needed)
+  try {
+    // node-gyp-build handles platform/arch detection automatically
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    const build = require('node-gyp-build');
+    return build(__dirname + '/../../') as NativeAddon;
+  } catch {
+    // Fall back to explicit build path
+    try {
+      // eslint-disable-next-line @typescript-eslint/no-var-requires
+      return require('../../build/Release/nodepyx_addon.node') as NativeAddon;
+    } catch {
+      // eslint-disable-next-line @typescript-eslint/no-var-requires
+      return require('../../build/Debug/nodepyx_addon.node') as NativeAddon;
+    }
+  }
+}
+