devspy-tool 1.0.0

package/puppeteer/explorer.js

@@ -0,0 +1,507 @@
+/**
+ * InteractionEngine — Systematic UI exploration.
+ *
+ * Designed to trigger every API call a real user might cause by:
+ *   Phase 1: Navigate SPA routes (sidebar links, nav tabs, menu items)
+ *   Phase 2: Interact with page content (tabs, accordions, cards, dropdowns)
+ *   Phase 3: Scroll to trigger lazy-loaded content
+ *   Phase 4: Trigger data-fetching patterns (search, filters, pagination)
+ *
+ * Safety: never clicks logout, delete, or destructive elements.
+ * Stability: every interaction is wrapped in try/catch with timeouts.
+ */
+
+// ─── Safety ─────────────────────────────────────────────────────────────────
+
+const DANGEROUS_KEYWORDS = [
+  "logout", "log out", "log-out",
+  "sign out", "sign-out", "signout",
+  "delete", "remove", "destroy",
+  "cancel subscription", "unsubscribe",
+  "deactivate", "close account", "terminate",
+  "reset password", "change password",
+];
+
+function isDangerous(text) {
+  const lower = (text || "").toLowerCase().trim();
+  return DANGEROUS_KEYWORDS.some((kw) => lower.includes(kw));
+}
+
+function isHidden(el) {
+  const style = window.getComputedStyle(el);
+  return (
+    style.display === "none" ||
+    style.visibility === "hidden" ||
+    style.opacity === "0" ||
+    el.offsetWidth === 0 ||
+    el.offsetHeight === 0
+  );
+}
+
+// ─── Scroll Engine ──────────────────────────────────────────────────────────
+
+/**
+ * Scroll the page in steps to trigger lazy-loaded content.
+ * Returns early if scroll height stops changing.
+ */
+async function intelligentScroll(page) {
+  try {
+    await page.evaluate(async () => {
+      const delay = (ms) => new Promise((r) => setTimeout(r, ms));
+      let previousHeight = 0;
+      let currentHeight = document.body.scrollHeight;
+      const step = Math.max(300, Math.floor(currentHeight / 6));
+
+      // Scroll down in steps
+      for (let y = 0; y < currentHeight; y += step) {
+        window.scrollTo({ top: y, behavior: "smooth" });
+        await delay(350);
+      }
+
+      // Scroll to absolute bottom
+      window.scrollTo({ top: currentHeight, behavior: "smooth" });
+      await delay(600);
+
+      // Check if page grew (infinite scroll)
+      const newHeight = document.body.scrollHeight;
+      if (newHeight > currentHeight) {
+        // One more round for the new content
+        for (let y = currentHeight; y < newHeight; y += step) {
+          window.scrollTo({ top: y, behavior: "smooth" });
+          await delay(350);
+        }
+        window.scrollTo({ top: newHeight, behavior: "smooth" });
+        await delay(400);
+      }
+
+      // Back to top
+      window.scrollTo({ top: 0, behavior: "smooth" });
+      await delay(300);
+    });
+  } catch {
+    // Page may have navigated during scroll — safe to ignore
+  }
+}
+
+// ─── Route Explorer (Phase 1) ───────────────────────────────────────────────
+
+/**
+ * Discover and click all navigation-like elements: <a>, sidebar <button>,
+ * tab controls, menu items, React Router <Link> components.
+ * Clicks each via the DOM (SPA-safe — no page.goto).
+ */
+async function exploreRoutes(page, network, maxPages = 8) {
+  console.log("[Explorer] Phase 1: Navigating SPA routes...");
+  const startCount = network.count;
+  const visitedPaths = new Set();
+
+  try {
+    visitedPaths.add(new URL(page.url()).pathname);
+  } catch {}
+
+  try {
+    const origin = await page.evaluate(() => window.location.origin);
+
+    // Gather every plausible navigation element
+    const navItems = await page.evaluate((orig) => {
+      const items = [];
+      const seen = new Set();
+
+      const selectors = [
+        "a[href]",
+        "nav button", "nav [role='button']",
+        "[class*='sidebar'] a", "[class*='sidebar'] button",
+        "[class*='Sidebar'] a", "[class*='Sidebar'] button",
+        "[class*='nav'] a", "[class*='nav'] button",
+        "[class*='Nav'] a", "[class*='Nav'] button",
+        "[role='navigation'] a", "[role='navigation'] button",
+        "[role='tab']", "[role='menuitem']",
+        "[class*='menu'] a", "[class*='menu'] button",
+        "[class*='Menu'] a", "[class*='Menu'] button",
+      ];
+
+      for (const sel of selectors) {
+        try {
+          for (const el of document.querySelectorAll(sel)) {
+            const text = (el.innerText || "").trim().slice(0, 80);
+            if (!text) continue;
+            const href = el.href || "";
+
+            // Skip external, javascript:, and hash-only links
+            if (href && !href.startsWith(orig) && !href.startsWith("/") && !href.startsWith("#")) continue;
+            if (href.startsWith("javascript:")) continue;
+
+            // Derive a unique key from the pathname or text
+            let key;
+            try {
+              key = href ? new URL(href, orig).pathname : `__text__${text}`;
+            } catch {
+              key = `__text__${text}`;
+            }
+            if (seen.has(key)) continue;
+            seen.add(key);
+
+            // Build a stable re-find strategy: id > unique selector > text
+            let findBy = null;
+            if (el.id) {
+              findBy = { type: "id", value: el.id };
+            } else {
+              findBy = { type: "text", value: text, tag: el.tagName.toLowerCase() };
+            }
+
+            items.push({ text, href, key, findBy });
+          }
+        } catch { /* selector failed — skip */ }
+      }
+      return items;
+    }, origin);
+
+    // Filter out dangerous and already-visited
+    const safe = navItems.filter((item) => {
+      if (isDangerous(item.text)) return false;
+      if (visitedPaths.has(item.key)) return false;
+      return true;
+    });
+
+    const toVisit = safe.slice(0, maxPages);
+    console.log(`[Explorer]   Found ${navItems.length} nav elements → visiting ${toVisit.length}`);
+
+    for (const item of toVisit) {
+      try {
+        console.log(`[Explorer]   → "${item.text}" (${item.key})`);
+
+        const clicked = await page.evaluate((findBy, txt) => {
+          let el = null;
+          if (findBy.type === "id") {
+            el = document.getElementById(findBy.value);
+          }
+          if (!el) {
+            // Text-based fallback
+            const tag = findBy.tag || "*";
+            el = Array.from(document.querySelectorAll(`${tag}, a, button, [role='tab'], [role='menuitem']`))
+              .find((e) => (e.innerText || "").trim() === txt);
+          }
+          if (el) { el.click(); return true; }
+          return false;
+        }, item.findBy, item.text);
+
+        if (!clicked) continue;
+
+        await sleep(1200);
+        await network.waitForSilence(2000, 8000);
+        await intelligentScroll(page);
+        await network.waitForSilence(1500, 4000);
+
+        visitedPaths.add(item.key);
+        try { visitedPaths.add(new URL(page.url()).pathname); } catch {}
+
+        console.log(`[Explorer]   ✓ ${page.url()} | +${network.count - startCount} new APIs`);
+      } catch (err) {
+        console.warn(`[Explorer]   ✗ "${item.text}" failed: ${err.message}`);
+      }
+    }
+  } catch (err) {
+    console.warn("[Explorer] Route exploration error:", err.message);
+  }
+
+  return visitedPaths;
+}
+
+// ─── Content Interactor (Phase 2) ───────────────────────────────────────────
+
+/**
+ * On the current page, interact with tabs, accordions, cards, dropdowns,
+ * modals, "load more" buttons — anything that might trigger API calls.
+ */
+async function interactWithContent(page, network) {
+  console.log("[Explorer] Phase 2: Interacting with page content...");
+  const startCount = network.count;
+
+  // --- 2a: Click tab-like elements ---
+  await clickElements(page, network, "tabs", [
+    "[role='tab']:not([aria-selected='true'])",
+    "button[data-tab]", "a[data-tab]",
+    ".tab:not(.active)", ".tab-item:not(.active)",
+  ], 6);
+
+  // --- 2b: Expand accordions / collapsed sections ---
+  await clickElements(page, network, "accordions", [
+    "[aria-expanded='false']",
+    "details > summary",
+    "[class*='accordion'] button",
+    "[class*='Accordion'] button",
+    "[class*='collapse'] button",
+    "[class*='Collapse'] button",
+  ], 5);
+
+  // --- 2c: Click data cards / list rows that look interactive ---
+  await clickElements(page, network, "cards", [
+    "[class*='card'][role='button']", "[class*='Card'][role='button']",
+    "tr[role='button']", "tr[class*='click']",
+    "[class*='list-item'][role='button']",
+    "[class*='row'][role='button']",
+  ], 4);
+
+  // --- 2d: Click "Load more" / "Show more" / pagination ---
+  try {
+    const loadMoreCount = await page.evaluate(() => {
+      const keywords = ["load more", "show more", "view all", "see all", "next page", "show all", "read more"];
+      let count = 0;
+      for (const el of document.querySelectorAll("button, a, [role='button']")) {
+        const text = (el.innerText || "").toLowerCase().trim();
+        if (keywords.some((kw) => text.includes(kw))) {
+          el.click();
+          count++;
+          if (count >= 4) break;
+        }
+      }
+      return count;
+    });
+    if (loadMoreCount > 0) {
+      console.log(`[Explorer]   "Load more" × ${loadMoreCount}`);
+      await sleep(800);
+      await network.waitForSilence(2000, 5000);
+    }
+  } catch { /* no load-more buttons */ }
+
+  // --- 2e: Open dropdown / select menus ---
+  await clickElements(page, network, "dropdowns", [
+    "[class*='dropdown'] button", "[class*='Dropdown'] button",
+    "[role='combobox']",
+    "[class*='filter'] button", "[class*='Filter'] button",
+  ], 3, async (page) => {
+    // After opening a dropdown, click the first option
+    try {
+      await page.evaluate(() => {
+        const opt = document.querySelector(
+          "[role='option'], [role='menuitem'], [class*='dropdown'] li, [class*='option']"
+        );
+        if (opt) opt.click();
+      });
+      await sleep(400);
+    } catch {}
+  });
+
+  // --- 2f: Click generic interactive buttons (not in nav) ---
+  try {
+    const buttons = await page.evaluate(() => {
+      const results = [];
+      for (const btn of document.querySelectorAll("main button, [class*='content'] button, section button")) {
+        const text = (btn.innerText || "").trim();
+        if (!text || text.length > 40) continue;
+        // Skip buttons we already handled (tabs, accordions, dropdowns, nav)
+        if (btn.closest("nav, [role='navigation'], [role='tablist'], [class*='dropdown']")) continue;
+        const style = window.getComputedStyle(btn);
+        if (style.display === "none" || style.visibility === "hidden") continue;
+        if (btn.id) results.push({ findBy: "id", value: btn.id, text });
+        else results.push({ findBy: "text", value: text, text });
+      }
+      return results.slice(0, 6);
+    });
+
+    for (const btn of buttons) {
+      if (isDangerous(btn.text)) continue;
+      try {
+        console.log(`[Explorer]   Button: "${btn.text.slice(0, 40)}"`);
+        await page.evaluate((b) => {
+          let el = b.findBy === "id" ? document.getElementById(b.value) : null;
+          if (!el) {
+            el = Array.from(document.querySelectorAll("button"))
+              .find((e) => (e.innerText || "").trim() === b.value);
+          }
+          if (el) el.click();
+        }, btn);
+        await sleep(600);
+        await network.waitForSilence(1500, 4000);
+      } catch {}
+    }
+  } catch {}
+
+  console.log(`[Explorer]   Phase 2 done: +${network.count - startCount} new APIs`);
+}
+
+// ─── Data Pattern Triggers (Phase 3) ────────────────────────────────────────
+
+/**
+ * Type into search inputs, toggle filters, click refresh buttons
+ * to trigger data-fetching API calls.
+ */
+async function triggerDataPatterns(page, network) {
+  console.log("[Explorer] Phase 3: Triggering data patterns...");
+  const startCount = network.count;
+
+  // --- 3a: Type into search inputs ---
+  try {
+    const searchInputs = await page.$$(
+      "input[type='search'], input[placeholder*='search' i], " +
+      "input[name*='search' i], input[class*='search' i], " +
+      "input[aria-label*='search' i]"
+    );
+    for (const input of searchInputs.slice(0, 2)) {
+      console.log("[Explorer]   Typing in search...");
+      await input.click();
+      await input.type("a", { delay: 120 });
+      await sleep(600);
+      await network.waitForSilence(1500, 4000);
+      // Clear and try another character
+      await input.click({ clickCount: 3 });
+      await input.press("Backspace");
+      await sleep(400);
+    }
+  } catch {}
+
+  // --- 3b: Click refresh / reload / sync buttons ---
+  try {
+    const clicked = await page.evaluate(() => {
+      const keywords = ["refresh", "reload", "retry", "sync"];
+      for (const el of document.querySelectorAll("button, [role='button']")) {
+        const text = (el.innerText || "").toLowerCase().trim();
+        const label = (el.getAttribute("aria-label") || "").toLowerCase();
+        if (keywords.some((kw) => text.includes(kw) || label.includes(kw))) {
+          el.click();
+          return true;
+        }
+      }
+      return false;
+    });
+    if (clicked) {
+      console.log("[Explorer]   Clicked refresh");
+      await network.waitForSilence(2000, 5000);
+    }
+  } catch {}
+
+  // --- 3c: Toggle filter checkboxes ---
+  try {
+    const checkboxes = await page.$$(
+      "[class*='filter'] input[type='checkbox'], " +
+      "[class*='Filter'] input[type='checkbox']"
+    );
+    for (const cb of checkboxes.slice(0, 3)) {
+      console.log("[Explorer]   Toggling filter checkbox");
+      await cb.click();
+      await sleep(400);
+      await network.waitForSilence(1500, 3000);
+    }
+  } catch {}
+
+  // --- 3d: Click pagination next/prev ---
+  try {
+    const pagClicked = await page.evaluate(() => {
+      for (const el of document.querySelectorAll("button, a, [role='button']")) {
+        const text = (el.innerText || "").trim().toLowerCase();
+        const label = (el.getAttribute("aria-label") || "").toLowerCase();
+        if (text === "next" || text === ">" || text === "›" || label.includes("next page")) {
+          el.click();
+          return true;
+        }
+      }
+      return false;
+    });
+    if (pagClicked) {
+      console.log("[Explorer]   Clicked pagination next");
+      await sleep(600);
+      await network.waitForSilence(2000, 5000);
+    }
+  } catch {}
+
+  console.log(`[Explorer]   Phase 3 done: +${network.count - startCount} new APIs`);
+}
+
+// ─── Main Orchestrator ──────────────────────────────────────────────────────
+
+/**
+ * Run full exploration: routes → content interactions → scroll → data patterns.
+ *
+ * @param {import('puppeteer-core').Page} page
+ * @param {import('./network').NetworkCollector} network
+ * @param {'basic'|'deep'} mode - 'basic' = Phase 1 only, 'deep' = all phases
+ * @param {number} maxRoutes - Maximum number of routes to explore
+ */
+async function explore(page, network, mode = "basic", maxRoutes = 8) {
+  const beforeCount = network.count;
+  console.log(`[Explorer] Starting ${mode} exploration...`);
+
+  // Phase 1: Always run — navigate SPA routes
+  const visitedPaths = await exploreRoutes(page, network, maxRoutes);
+  console.log(`[Explorer] After routes: ${network.count} total (was ${beforeCount})`);
+
+  // Phase 2: Deep only — interact with content on current page
+  if (mode === "deep") {
+    await interactWithContent(page, network);
+
+    // Go back to main page and interact there too
+    if (visitedPaths.size > 1) {
+      try {
+        await page.goBack();
+        await sleep(1000);
+        await network.waitForSilence(1500, 4000);
+        await interactWithContent(page, network);
+      } catch {}
+    }
+  }
+
+  // Phase 3: Always run — scroll for lazy-loaded content
+  await intelligentScroll(page);
+  await network.waitForSilence(2000, 5000);
+
+  // Phase 4: Deep only — trigger data patterns
+  if (mode === "deep") {
+    await triggerDataPatterns(page, network);
+  }
+
+  const newCount = network.count - beforeCount;
+  console.log(`[Explorer] Exploration complete: discovered ${newCount} new API calls`);
+  return newCount;
+}
+
+// ─── Utility Functions ──────────────────────────────────────────────────────
+
+/**
+ * Click elements matching the given selectors, safely.
+ * @param {number} maxPerSelector - Max elements to click per selector group
+ * @param {Function} [afterClick] - Optional callback after each click
+ */
+async function clickElements(page, network, label, selectors, maxTotal = 5, afterClick = null) {
+  let clicked = 0;
+  for (const sel of selectors) {
+    if (clicked >= maxTotal) break;
+    try {
+      const elements = await page.$$(sel);
+      for (const el of elements) {
+        if (clicked >= maxTotal) break;
+        try {
+          const text = await page.evaluate((e) => {
+            const style = window.getComputedStyle(e);
+            if (style.display === "none" || style.visibility === "hidden") return null;
+            return (e.innerText || "").trim().slice(0, 60);
+          }, el);
+
+          if (text === null) continue; // hidden
+          if (isDangerous(text)) continue;
+
+          console.log(`[Explorer]   ${label}: "${text.slice(0, 40)}"`);
+          await el.click().catch(() => {});
+          clicked++;
+          await sleep(600);
+          await network.waitForSilence(1500, 3500);
+          if (afterClick) await afterClick(page);
+        } catch { /* element stale or detached */ }
+      }
+    } catch { /* selector invalid or not found */ }
+  }
+  return clicked;
+}
+
+function sleep(ms) {
+  return new Promise((r) => setTimeout(r, ms));
+}
+
+module.exports = {
+  explore,
+  exploreRoutes,
+  interactWithContent,
+  triggerDataPatterns,
+  intelligentScroll,
+  isDangerous,
+  sleep,
+};