bluera-knowledge 0.13.3 → 0.14.1

package/src/crawl/bridge.ts

@@ -1,339 +0,0 @@
-import { spawn, type ChildProcess } from 'node:child_process';
-import { randomUUID } from 'node:crypto';
-import path from 'node:path';
-import { createInterface, type Interface as ReadlineInterface } from 'node:readline';
-import { fileURLToPath } from 'node:url';
-import { ZodError } from 'zod';
-import {
-  type CrawlResult,
-  type HeadlessResult,
-  type ParsePythonResult,
-  type CrawledLink,
-  validateCrawlResult,
-  validateHeadlessResult,
-  validateParsePythonResult,
-} from './schemas.js';
-import { createLogger } from '../logging/index.js';
-
-const logger = createLogger('python-bridge');
-
-// Re-export for backwards compatibility
-export type { CrawledLink, ParsePythonResult };
-
-type PendingResult = CrawlResult | HeadlessResult | ParsePythonResult;
-
-interface PendingRequest {
-  resolve: (v: PendingResult) => void;
-  reject: (e: Error) => void;
-  timeout: NodeJS.Timeout;
-  method: 'crawl' | 'fetch_headless' | 'parse_python';
-}
-
-export class PythonBridge {
-  private process: ChildProcess | null = null;
-  private readonly pending: Map<string, PendingRequest> = new Map();
-  private stoppingIntentionally = false;
-  private stdoutReadline: ReadlineInterface | null = null;
-  private stderrReadline: ReadlineInterface | null = null;
-
-  start(): Promise<void> {
-    if (this.process) return Promise.resolve();
-
-    // Compute absolute path to Python worker using import.meta.url
-    // This works both in development (src/) and production (dist/)
-    const currentFilePath = fileURLToPath(import.meta.url);
-    const isProduction = currentFilePath.includes('/dist/');
-
-    let pythonWorkerPath: string;
-    if (isProduction) {
-      // Production: Find dist dir and go to sibling python/ directory
-      const distIndex = currentFilePath.indexOf('/dist/');
-      const pluginRoot = currentFilePath.substring(0, distIndex);
-      pythonWorkerPath = path.join(pluginRoot, 'python', 'crawl_worker.py');
-    } else {
-      // Development: Go up from src/crawl to find python/
-      const srcDir = path.dirname(path.dirname(currentFilePath));
-      const projectRoot = path.dirname(srcDir);
-      pythonWorkerPath = path.join(projectRoot, 'python', 'crawl_worker.py');
-    }
-
-    logger.debug(
-      { pythonWorkerPath, currentFilePath, isProduction },
-      'Starting Python bridge process'
-    );
-
-    this.process = spawn('python3', [pythonWorkerPath], {
-      stdio: ['pipe', 'pipe', 'pipe'],
-    });
-
-    // Add error handler for process spawn errors
-    this.process.on('error', (err) => {
-      logger.error({ error: err.message, stack: err.stack }, 'Python bridge process error');
-      this.rejectAllPending(new Error(`Process error: ${err.message}`));
-    });
-
-    // Add exit handler to detect non-zero exits
-    this.process.on('exit', (code, signal) => {
-      if (code !== 0 && code !== null) {
-        logger.error({ code }, 'Python bridge process exited with non-zero code');
-        this.rejectAllPending(new Error(`Process exited with code ${String(code)}`));
-      } else if (signal && !this.stoppingIntentionally) {
-        // Only log if we didn't intentionally stop the process
-        logger.error({ signal }, 'Python bridge process killed with signal');
-        this.rejectAllPending(new Error(`Process killed with signal ${signal}`));
-      }
-      this.process = null;
-      this.stoppingIntentionally = false;
-    });
-
-    // Add stderr logging
-    if (this.process.stderr) {
-      this.stderrReadline = createInterface({ input: this.process.stderr });
-      this.stderrReadline.on('line', (line) => {
-        logger.warn({ stderr: line }, 'Python bridge stderr output');
-      });
-    }
-
-    if (this.process.stdout === null) {
-      this.process.kill(); // Kill process to prevent zombie
-      this.process = null; // Clean up reference
-      return Promise.reject(new Error('Python bridge process stdout is null'));
-    }
-    this.stdoutReadline = createInterface({ input: this.process.stdout });
-    this.stdoutReadline.on('line', (line) => {
-      // Filter out non-JSON lines (crawl4ai verbose output)
-      if (!line.trim().startsWith('{')) {
-        return;
-      }
-
-      try {
-        // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
-        const response = JSON.parse(line) as {
-          id: string;
-          error?: { message: string };
-          result?: PendingResult;
-        };
-        const pending = this.pending.get(response.id);
-        if (pending !== undefined) {
-          if (response.error !== undefined) {
-            clearTimeout(pending.timeout);
-            this.pending.delete(response.id);
-            pending.reject(new Error(response.error.message));
-          } else if (response.result !== undefined) {
-            clearTimeout(pending.timeout);
-            this.pending.delete(response.id);
-
-            // Validate response structure based on method type
-            try {
-              let validated: PendingResult;
-              if (pending.method === 'crawl') {
-                validated = validateCrawlResult(response.result);
-              } else if (pending.method === 'fetch_headless') {
-                validated = validateHeadlessResult(response.result);
-              } else {
-                validated = validateParsePythonResult(response.result);
-              }
-              pending.resolve(validated);
-            } catch (error: unknown) {
-              // Log validation failure with original response for debugging
-              if (error instanceof ZodError) {
-                logger.error(
-                  {
-                    issues: error.issues,
-                    response: JSON.stringify(response.result),
-                  },
-                  'Python bridge response validation failed'
-                );
-                pending.reject(
-                  new Error(`Invalid response format from Python bridge: ${error.message}`)
-                );
-              } else {
-                const errorMessage = error instanceof Error ? error.message : String(error);
-                logger.error({ error: errorMessage }, 'Response validation error');
-                pending.reject(new Error(`Response validation error: ${errorMessage}`));
-              }
-            }
-          }
-          // If neither result nor error, leave pending (will timeout)
-        }
-      } catch (err) {
-        logger.error(
-          {
-            error: err instanceof Error ? err.message : String(err),
-            line,
-          },
-          'Failed to parse JSON response from Python bridge'
-        );
-      }
-    });
-
-    return Promise.resolve();
-  }
-
-  async crawl(url: string, timeoutMs: number = 30000): Promise<CrawlResult> {
-    if (!this.process) await this.start();
-
-    const id = randomUUID();
-    const request = {
-      jsonrpc: '2.0',
-      id,
-      method: 'crawl',
-      params: { url },
-    };
-
-    return new Promise((resolve, reject) => {
-      // Set up timeout
-      const timeout = setTimeout(() => {
-        const pending = this.pending.get(id);
-        if (pending) {
-          this.pending.delete(id);
-          reject(new Error(`Crawl timeout after ${String(timeoutMs)}ms for URL: ${url}`));
-        }
-      }, timeoutMs);
-
-      this.pending.set(id, {
-        // eslint-disable-next-line @typescript-eslint/consistent-type-assertions -- Promise resolve type narrowing
-        resolve: resolve as (v: PendingResult) => void,
-        reject,
-        timeout,
-        method: 'crawl',
-      });
-      if (!this.process?.stdin) {
-        reject(new Error('Python bridge process not available'));
-        return;
-      }
-      this.process.stdin.write(`${JSON.stringify(request)}\n`);
-    });
-  }
-
-  async fetchHeadless(url: string, timeoutMs: number = 60000): Promise<HeadlessResult> {
-    if (!this.process) await this.start();
-
-    const id = randomUUID();
-    const request = {
-      jsonrpc: '2.0',
-      id,
-      method: 'fetch_headless',
-      params: { url },
-    };
-
-    return new Promise((resolve, reject) => {
-      const timeout = setTimeout(() => {
-        const pending = this.pending.get(id);
-        if (pending) {
-          this.pending.delete(id);
-          reject(new Error(`Headless fetch timeout after ${String(timeoutMs)}ms for URL: ${url}`));
-        }
-      }, timeoutMs);
-
-      this.pending.set(id, {
-        // eslint-disable-next-line @typescript-eslint/consistent-type-assertions -- Promise resolve type narrowing
-        resolve: resolve as (v: PendingResult) => void,
-        reject,
-        timeout,
-        method: 'fetch_headless',
-      });
-      if (!this.process?.stdin) {
-        reject(new Error('Python bridge process not available'));
-        return;
-      }
-      this.process.stdin.write(`${JSON.stringify(request)}\n`);
-    });
-  }
-
-  async parsePython(
-    code: string,
-    filePath: string,
-    timeoutMs: number = 10000
-  ): Promise<ParsePythonResult> {
-    if (!this.process) await this.start();
-
-    const id = randomUUID();
-    const request = {
-      jsonrpc: '2.0',
-      id,
-      method: 'parse_python',
-      params: { code, filePath },
-    };
-
-    return new Promise((resolve, reject) => {
-      const timeout = setTimeout(() => {
-        const pending = this.pending.get(id);
-        if (pending) {
-          this.pending.delete(id);
-          reject(
-            new Error(`Python parsing timeout after ${String(timeoutMs)}ms for file: ${filePath}`)
-          );
-        }
-      }, timeoutMs);
-
-      this.pending.set(id, {
-        // eslint-disable-next-line @typescript-eslint/consistent-type-assertions -- Promise resolve type narrowing
-        resolve: resolve as (v: PendingResult) => void,
-        reject,
-        timeout,
-        method: 'parse_python',
-      });
-      if (!this.process?.stdin) {
-        reject(new Error('Python bridge process not available'));
-        return;
-      }
-      this.process.stdin.write(`${JSON.stringify(request)}\n`);
-    });
-  }
-
-  stop(): Promise<void> {
-    if (!this.process) {
-      return Promise.resolve();
-    }
-
-    return new Promise((resolve) => {
-      this.stoppingIntentionally = true;
-      this.rejectAllPending(new Error('Python bridge stopped'));
-
-      // Close readline interfaces to prevent resource leaks
-      if (this.stdoutReadline) {
-        this.stdoutReadline.close();
-        this.stdoutReadline = null;
-      }
-      if (this.stderrReadline) {
-        this.stderrReadline.close();
-        this.stderrReadline = null;
-      }
-
-      // Wait for process to actually exit before resolving
-      const proc = this.process;
-      if (proc === null) {
-        resolve();
-        return;
-      }
-
-      // Set up exit handler to resolve when process terminates
-      const onExit = (): void => {
-        resolve();
-      };
-      proc.once('exit', onExit);
-
-      // Send SIGTERM to gracefully stop
-      proc.kill();
-
-      // Safety timeout in case process doesn't exit within 1 second
-      setTimeout(() => {
-        proc.removeListener('exit', onExit);
-        if (this.process === proc) {
-          proc.kill('SIGKILL'); // Force kill
-          this.process = null;
-        }
-        resolve();
-      }, 1000);
-    });
-  }
-
-  private rejectAllPending(error: Error): void {
-    for (const pending of this.pending.values()) {
-      clearTimeout(pending.timeout);
-      pending.reject(error);
-    }
-    this.pending.clear();
-  }
-}