devspy-tool 1.0.0

package/puppeteer/interceptor.js

@@ -0,0 +1,622 @@
+/**
+ * SessionEngine — Full-session API capture orchestrator.
+ *
+ * Architecture:
+ *   1. Launch browser (headless or visible for manual mode)
+ *   2. Start CDP network capture (runs continuously until stop)
+ *   3. Load target URL
+ *   4. Optional: run login flow
+ *   5. Optional: navigate to post-login URL (SPA-safe)
+ *   6. Run exploration engine (routes, interactions, scroll, data patterns)
+ *   7. Collect final results
+ *
+ * Manual mode:
+ *   - Browser opens visually (headless: false)
+ *   - Network capture streams results via EventEmitter
+ *   - Session stays alive until explicitly stopped
+ *
+ * The network collector runs CONTINUOUSLY from step 2 → 7, capturing:
+ *   - Page load APIs
+ *   - Login APIs
+ *   - Post-redirect APIs
+ *   - Interaction-triggered APIs
+ *   - Lazy-loaded APIs
+ *   - Background/polling APIs
+ */
+
+const puppeteer = require("puppeteer-core");
+const EventEmitter = require("events");
+const { CHROME_PATH, RESPONSE_PREVIEW_LIMIT } = require("../config");
+const { NetworkCollector } = require("./network");
+const { explore, intelligentScroll, isDangerous, sleep } = require("./explorer");
+
+// ─── Login Handler ──────────────────────────────────────────────────────────
+
+/**
+ * Try the primary selector first, then fall back through alternatives.
+ * Includes "smart" heuristics to detect inputs by type, label, or placeholder.
+ */
+async function resolveSelector(page, primary, fallbacks = [], type = "any") {
+  // 1. Try user-provided selector
+  if (primary) {
+    try {
+      const el = await page.waitForSelector(primary, { timeout: 3000 });
+      if (el) return el;
+    } catch { /* ignore */ }
+  }
+
+  // 2. Try fallbacks
+  for (const selector of fallbacks) {
+    try {
+      if (selector.includes(":has-text(")) {
+        const textMatch = selector.match(/:has-text\('(.+?)'\)/);
+        if (textMatch) {
+          const tag = selector.split(":")[0] || "*";
+          const text = textMatch[1].toLowerCase();
+          const el = await page.evaluateHandle((t, txt) => {
+            return Array.from(document.querySelectorAll(t)).find(e =>
+              e.innerText?.toLowerCase().includes(txt) || e.value?.toLowerCase().includes(txt)
+            ) || null;
+          }, tag, text);
+          const element = el.asElement();
+          if (element) return element;
+        }
+        continue;
+      }
+      const el = await page.$(selector);
+      if (el) return el;
+    } catch { continue; }
+  }
+
+  // 3. Smart Heuristics based on "type"
+  return await page.evaluateHandle((t) => {
+    const isVisible = (el) => !!(el.offsetWidth || el.offsetHeight || el.getClientRects().length);
+    const inputs = Array.from(document.querySelectorAll("input, button, [role='button']")).filter(isVisible);
+
+    if (t === "email") {
+      return inputs.find(i =>
+        i.type === "email" ||
+        i.name?.toLowerCase().includes("email") ||
+        i.id?.toLowerCase().includes("email") ||
+        i.placeholder?.toLowerCase().includes("email")
+      ) || inputs.find(i => i.type === "text" && !i.name?.toLowerCase().includes("search")) || null;
+    }
+
+    if (t === "password") {
+      return inputs.find(i => i.type === "password" || i.name?.toLowerCase().includes("password")) || null;
+    }
+
+    if (t === "submit") {
+      return inputs.find(i =>
+        i.type === "submit" ||
+        ["login", "sign in", "submit", "continue"].some(k => i.innerText?.toLowerCase().includes(k) || i.value?.toLowerCase().includes(k))
+      ) || null;
+    }
+
+    return null;
+  }, type).then(h => h.asElement());
+}
+
+/**
+ * Execute the Smart Login flow (9-step system):
+ * 1. Open page (handled by caller)
+ * 2. Check login form presence
+ * 3. If not → click login button keywords
+ * 4. Wait for inputs to appear
+ * 5. Smartly detect email/password
+ * 6. Fill credentials
+ * 7. Click submit
+ * 8. Wait for session stability / navigation
+ * 9. Verify success
+ */
+async function performLogin(page, network, loginConfig) {
+  console.log("[SmartLogin] Starting flow...");
+
+  // Step 2 & 3: Check for form or click navigation
+  const hasInputs = await page.evaluate(() => {
+    return !!document.querySelector("input[type='password'], input[name*='password']");
+  });
+
+  if (!hasInputs) {
+    console.log("[SmartLogin] No form visible. Searching for login trigger...");
+    const clicked = await page.evaluate(() => {
+      const primaryKeywords = ["login", "sign in", "log in", "enter portal", "access", "sign-in"];
+      const secondaryKeywords = ["get started", "start now", "join", "try", "go to app", "dashboard"];
+      
+      const links = Array.from(document.querySelectorAll("a, button, [role='button']"));
+      
+      const findMatch = (keys) => links.find(el => {
+        const txt = el.innerText?.toLowerCase() || "";
+        return keys.some(k => txt.includes(k));
+      });
+
+      // Try primary (Login) first, then secondary (Get Started)
+      const trigger = findMatch(primaryKeywords) || findMatch(secondaryKeywords);
+      
+      if (trigger) {
+        trigger.click();
+        return { text: trigger.innerText?.trim(), id: trigger.id || "no-id" };
+      }
+      return null;
+    });
+
+    if (clicked) {
+      console.log(`[SmartLogin] Trigger clicked: "${clicked.text}" (${clicked.id}). Waiting for form...`);
+      await sleep(2000);
+      await network.waitForSilence(1000, 5000);
+    }
+  }
+
+  // Step 5: Detect Email
+  console.log("[SmartLogin] Detecting email field...");
+  const emailEl = await resolveSelector(page, loginConfig.emailSelector, [
+    "input[type='email']", "input[name='email']", "input[id*='email' i]"
+  ], "email");
+
+  // Step 5: Detect Password
+  console.log("[SmartLogin] Detecting password field...");
+  const passwordEl = await resolveSelector(page, loginConfig.passwordSelector, [
+    "input[type='password']", "input[name='password']"
+  ], "password");
+
+  if (!emailEl || !passwordEl) {
+    throw new Error(`Login fields not found. Email: ${!!emailEl}, Password: ${!!passwordEl}`);
+  }
+
+  // Step 6: Fill credentials
+  console.log("[SmartLogin] Filling form...");
+  await emailEl.click({ clickCount: 3 });
+  await emailEl.type(loginConfig.email, { delay: 30 });
+  await passwordEl.click({ clickCount: 3 });
+  await passwordEl.type(loginConfig.password, { delay: 30 });
+
+  // Step 7: Submit
+  console.log("[SmartLogin] Submitting...");
+  const submitEl = await resolveSelector(page, loginConfig.submitSelector, [
+    "button[type='submit']", "input[type='submit']", "button:has-text('Login')"
+  ], "submit");
+
+  if (submitEl) {
+    await submitEl.click();
+  } else {
+    await passwordEl.press("Enter");
+  }
+
+  // Step 8: Wait for stability
+  console.log("[SmartLogin] Waiting for navigation/session setup...");
+  try {
+    await Promise.race([
+      page.waitForNavigation({ waitUntil: "networkidle2", timeout: 10000 }),
+      network.waitForSilence(3000, 10000)
+    ]);
+  } catch { /* session might be SPA-based */ }
+
+  const finalUrl = page.url();
+  console.log(`[SmartLogin] Flow finished. Current URL: ${finalUrl}`);
+
+  // Step 9: Simple success check
+  if (finalUrl.includes("login") || finalUrl.includes("signin")) {
+    console.warn("[SmartLogin] ⚠ Might still be on login page. Check credentials.");
+  }
+}
+
+/**
+ * Navigate to the post-login URL using SPA-safe method.
+ * Tries clicking a matching <a> first, falls back to page.goto().
+ */
+async function navigatePostLogin(page, network, targetUrl) {
+  console.log(`[Session] Navigating to post-login URL: ${targetUrl}`);
+
+  try {
+    const targetPath = new URL(targetUrl).pathname;
+
+    // Try SPA-safe click first
+    const clicked = await page.evaluate((path) => {
+      const link = Array.from(document.querySelectorAll("a[href]")).find((a) => {
+        try { return new URL(a.href).pathname === path; } catch { return false; }
+      });
+      if (link) { link.click(); return true; }
+      return false;
+    }, targetPath);
+
+    if (clicked) {
+      console.log(`[Session] Clicked SPA link for ${targetPath}`);
+      await sleep(2000);
+    } else {
+      // Fallback: direct navigation (Firebase auth persists in IndexedDB)
+      await page.goto(targetUrl, { waitUntil: "networkidle2", timeout: 20000 });
+    }
+
+    await network.waitForSilence(3000, 12000);
+    await intelligentScroll(page);
+    await network.waitForSilence(2000, 5000);
+
+    // Verify we're authenticated
+    const currentPath = new URL(page.url()).pathname;
+    if (currentPath.includes("login") || currentPath.includes("signin")) {
+      console.warn("[Session] ⚠ Post-login URL redirected to login page");
+    } else {
+      console.log(`[Session] Post-login URL loaded: ${page.url()}`);
+    }
+    console.log(`[Session] Captured: ${network.count} APIs`);
+  } catch (err) {
+    console.warn("[Session] Post-login navigation failed:", err.message);
+  }
+}
+
+// ─── SessionEngine Class ────────────────────────────────────────────────────
+
+class SessionEngine extends EventEmitter {
+  constructor() {
+    super();
+    this.browser = null;
+    this.page = null;
+    this.network = null;
+    this._running = false;
+  }
+
+  /**
+   * Run an automated scan: load page, login, explore, return results.
+   *
+   * @param {string} url - Target URL to scan
+   * @param {object} options
+   * @param {number} [options.waitTime=10000] - Max wait time for network silence after page load
+   * @param {object|null} [options.login] - Login credentials & selectors
+   * @param {boolean} [options.deepExplore=false] - Run full deep exploration
+   * @param {string|null} [options.postLoginUrl] - URL to navigate after login
+   * @returns {Promise<object[]>} Array of captured API requests
+   */
+  async runAutoScan(url, options = {}) {
+    const {
+      waitTime = 10000,
+      login = null,
+      deepExplore = false,
+      postLoginUrl = null,
+      keepAlive = false,
+    } = options;
+
+    // Auto-enable exploration when login is provided
+    const shouldExplore = !!login;
+    const mode = deepExplore ? "deep" : "basic";
+
+    console.log(`\n${"═".repeat(60)}`);
+    console.log(`[Session] Auto scan: ${url}`);
+    console.log(`[Session] login=${!!login} explore=${shouldExplore} deep=${deepExplore} wait=${waitTime}ms keepAlive=${keepAlive}`);
+    console.log(`${"═".repeat(60)}\n`);
+
+    try {
+      // ── Step 1: Launch browser ──
+      await this._launchBrowser(true);
+
+      // ── Step 2: Start continuous network capture ──
+      this.network = new NetworkCollector(this.page, {
+        responsePreviewLimit: RESPONSE_PREVIEW_LIMIT,
+      });
+      await this.network.start();
+      this._running = true;
+
+      // Forward network events for logging
+      this.network.on("request", (entry) => {
+        this.emit("request", entry);
+      });
+
+      // ── Step 3: Load target URL ──
+      console.log("[Session] Loading target URL...");
+      try {
+        await this.page.goto(url, { waitUntil: "networkidle2", timeout: 30000 });
+      } catch (err) {
+        console.warn("[Session] Page load warning:", err.message);
+      }
+
+      // Wait for initial API burst
+      console.log("[Session] Waiting for initial APIs...");
+      await this.network.waitForSilence(3000, 12000);
+      console.log(`[Session] Initial load: ${this.network.count} APIs captured`);
+
+      // ── Step 4: Login flow ──
+      if (login) {
+        await performLogin(this.page, this.network, login);
+      }
+
+      // ── Step 5: Post-login URL ──
+      if (postLoginUrl) {
+        await navigatePostLogin(this.page, this.network, postLoginUrl);
+      }
+
+      // ── Step 6: Scroll the current page ──
+      await intelligentScroll(this.page);
+      await this.network.waitForSilence(2000, 5000);
+      console.log(`[Session] After scroll: ${this.network.count} APIs captured`);
+
+      // ── Step 7: Exploration ──
+      if (shouldExplore) {
+        const maxRoutes = deepExplore ? 10 : 6;
+        await explore(this.page, this.network, mode, maxRoutes);
+      } else {
+        // Even without login, wait the full waitTime for any delayed APIs
+        await this.network.waitForSilence(3000, waitTime);
+      }
+
+      // ── Step 8: Final wait for trailing responses ──
+      await sleep(2000);
+      await this.network.waitForSilence(2000, 4000);
+
+      // ── Step 9: Collect results ──
+      const results = this.network.getResults();
+      console.log(`\n${"═".repeat(60)}`);
+      console.log(`[Session] DONE. Total: ${results.length} API calls captured.`);
+      console.log(`${"═".repeat(60)}\n`);
+
+      return results;
+    } finally {
+      // If keepAlive, stop the network collector but keep the browser open
+      if (keepAlive) {
+        console.log(`[Session] keepAlive=true — browser stays open for replay`);
+        if (this.network) {
+          await this.network.stop();
+        }
+        // Do NOT close browser or page
+      } else {
+        await this._cleanup();
+      }
+    }
+  }
+
+  /**
+   * Start a manual session: browser opens visually, user interacts,
+   * APIs are captured until stopManualSession() is called.
+   *
+   * @param {string} url - Target URL
+   * @param {object} options
+   * @param {object|null} [options.login] - Auto-login before handing control to user
+   * @param {string|null} [options.postLoginUrl] - Navigate here after login
+   * @returns {string} Session ID (used to stream results / stop)
+   */
+  async startManualSession(url, options = {}) {
+    const { login = null, postLoginUrl = null } = options;
+
+    console.log(`\n${"═".repeat(60)}`);
+    console.log(`[Session] Manual mode: ${url}`);
+    console.log(`${"═".repeat(60)}\n`);
+
+    // ── Launch visible browser ──
+    await this._launchBrowser(false);
+
+    // ── Start network capture ──
+    this.network = new NetworkCollector(this.page, {
+      responsePreviewLimit: RESPONSE_PREVIEW_LIMIT,
+    });
+    await this.network.start();
+    this._running = true;
+
+    // Forward events for SSE streaming
+    this.network.on("complete", (entry) => this.emit("apiCaptured", entry));
+    this.network.on("failed", (entry) => this.emit("apiCaptured", entry));
+
+    // ── Load page ──
+    try {
+      await this.page.goto(url, { waitUntil: "networkidle2", timeout: 30000 });
+    } catch (err) {
+      console.warn("[Session] Manual: page load warning:", err.message);
+    }
+
+    await this.network.waitForSilence(3000, 10000);
+    console.log(`[Session] Manual: page loaded, ${this.network.count} initial APIs`);
+
+    // ── Optional auto-login ──
+    if (login) {
+      await performLogin(this.page, this.network, login);
+      if (postLoginUrl) {
+        await navigatePostLogin(this.page, this.network, postLoginUrl);
+      }
+    }
+
+    console.log("[Session] Manual mode active — interact with the browser.");
+    console.log("[Session] APIs are being captured in real-time.");
+  }
+
+  /**
+   * Stop a manual session. If keepAlive is true, the browser stays
+   * open for replay; otherwise it is closed.
+   *
+   * @param {boolean} keepAlive - Keep browser open for replay
+   */
+  async stopManualSession(keepAlive = false) {
+    console.log("[Session] Stopping manual session...");
+    const results = this.network ? this.network.getResults() : [];
+
+    if (keepAlive) {
+      console.log(`[Session] keepAlive=true — browser stays open for replay`);
+      if (this.network) {
+        await this.network.stop();
+      }
+      this._running = false;
+    } else {
+      await this._cleanup();
+    }
+
+    console.log(`[Session] Manual session ended. Total: ${results.length} APIs captured.`);
+    return results;
+  }
+
+  /**
+   * Execute a fetch request INSIDE the Puppeteer page context.
+   * This carries the page's cookies, auth tokens, localStorage, IndexedDB,
+   * and service-worker state — making it work for authenticated APIs.
+   *
+   * @param {object} config
+   * @param {string} config.url
+   * @param {string} config.method
+   * @param {object} [config.headers]
+   * @param {string} [config.body]
+   * @returns {Promise<{status, statusText, headers, body}>}
+   */
+  async replayInContext(config) {
+    if (!this.page) {
+      throw new Error("No active browser session for replay");
+    }
+
+    // Check whether the page is still alive
+    try {
+      await this.page.evaluate(() => document.readyState);
+    } catch {
+      throw new Error("Browser session has been closed or crashed");
+    }
+
+    console.log(`[Replay] In-context: ${config.method} ${config.url}`);
+
+    const result = await this.page.evaluate(async (cfg) => {
+      try {
+        const fetchOpts = {
+          method: cfg.method || "GET",
+          headers: cfg.headers || {},
+          credentials: "include",   // ← sends cookies
+        };
+
+        // Attach body for non-GET/HEAD
+        if (cfg.body && cfg.method !== "GET" && cfg.method !== "HEAD") {
+          fetchOpts.body = cfg.body;
+        }
+
+        const response = await fetch(cfg.url, fetchOpts);
+
+        // Collect response headers
+        const respHeaders = {};
+        response.headers.forEach((value, key) => {
+          respHeaders[key] = value;
+        });
+
+        // Read body (limit to ~50KB to avoid memory issues)
+        let body = "";
+        try {
+          body = await response.text();
+          if (body.length > 50000) {
+            body = body.substring(0, 50000) + "\n... [truncated]";
+          }
+        } catch {
+          body = "[Could not read response body]";
+        }
+
+        return {
+          success: true,
+          status: response.status,
+          statusText: response.statusText,
+          headers: respHeaders,
+          body,
+        };
+      } catch (err) {
+        return {
+          success: false,
+          error: err.message || "Fetch failed inside browser context",
+        };
+      }
+    }, config);
+
+    if (!result.success) {
+      throw new Error(result.error);
+    }
+
+    console.log(`[Replay] Response: ${result.status} ${result.statusText}`);
+    return result;
+  }
+
+  /** Check if session is running. */
+  get isRunning() {
+    return this._running;
+  }
+
+  /** Get current capture count. */
+  get captureCount() {
+    return this.network ? this.network.count : 0;
+  }
+
+  /**
+   * Check if the browser/page is still usable for replay.
+   */
+  get isAlive() {
+    return !!(this.browser && this.page);
+  }
+
+  /**
+   * Explicitly close the browser session.
+   * Used by sessionStore cleanup.
+   */
+  async close() {
+    await this._cleanup();
+  }
+
+  // ─── Private ────────────────────────────────────────────────────────────
+
+  async _launchBrowser(headless = true) {
+    this.browser = await puppeteer.launch({
+      executablePath: CHROME_PATH,
+      headless: headless ? "new" : false,
+      args: [
+        "--no-sandbox",
+        "--disable-setuid-sandbox",
+        "--disable-blink-features=AutomationControlled",
+        "--disable-web-security",
+        "--disable-features=VizDisplayCompositor",
+        "--allow-insecure-localhost",
+        "--ignore-certificate-errors",
+      ],
+    });
+
+    this.page = await this.browser.newPage();
+
+    // Anti-detection
+    await this.page.evaluateOnNewDocument(() => {
+      Object.defineProperty(navigator, "webdriver", { get: () => false });
+    });
+
+    await this.page.setUserAgent(
+      "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " +
+      "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
+    );
+
+    await this.page.setViewport({ width: 1280, height: 800 });
+
+    // Handle unexpected dialogs
+    this.page.on("dialog", async (dialog) => {
+      try { await dialog.dismiss(); } catch {}
+    });
+  }
+
+  async _cleanup() {
+    this._running = false;
+    if (this.network) {
+      await this.network.stop();
+      this.network = null;
+    }
+    if (this.browser) {
+      try { await this.browser.close(); } catch {}
+      this.browser = null;
+      this.page = null;
+    }
+  }
+}
+
+// ─── Convenience Function (backward compatible) ─────────────────────────────
+
+/**
+ * Drop-in replacement for the old interceptRequests function.
+ * Creates a SessionEngine, runs an auto scan, returns results.
+ */
+async function interceptRequests(url, options = {}) {
+  const {
+    waitTime = 10000,
+    login = null,
+    explorePages = false,
+    postLoginUrl = null,
+  } = options;
+
+  const engine = new SessionEngine();
+  return engine.runAutoScan(url, {
+    waitTime,
+    login,
+    deepExplore: explorePages,
+    postLoginUrl,
+  });
+}
+
+module.exports = { SessionEngine, interceptRequests };

package/puppeteer/launcher.js

@@ -0,0 +1,30 @@
+const puppeteer = require("puppeteer-core");
+
+const CHROME_PATH = "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome";
+
+async function launchAndVisit(url) {
+    console.log(`[Puppeteer] Launching browser for: ${url}`);
+
+    const browser = await puppeteer.launch({
+        executablePath: CHROME_PATH,   // ← point to your local Chrome
+        headless: "new",
+        args: ["--no-sandbox", "--disable-setuid-sandbox"],
+    });
+
+    const page = await browser.newPage();
+
+    await page.goto(url, {
+        waitUntil: "networkidle2",
+        timeout: 30000,
+    });
+
+    const title = await page.title();
+    console.log(`[Puppeteer] Page title: "${title}"`);
+
+    await browser.close();
+    console.log("[Puppeteer] Browser closed.");
+
+    return { title };
+}
+
+launchAndVisit("https://example.com").then(console.log).catch(console.error);