devspy-tool 1.0.0

package/puppeteer/network.js

@@ -0,0 +1,253 @@
+/**
+ * NetworkCollector — CDP-based continuous network capture.
+ *
+ * Uses Chrome DevTools Protocol directly (not Puppeteer's request interception)
+ * to capture ALL network traffic including Service Worker requests,
+ * background fetches, WebSocket frames, and preflight CORS requests.
+ *
+ * Emits events: 'request', 'response', 'complete', 'failed'
+ * so callers can stream results in real-time (SSE / manual mode).
+ */
+
+const EventEmitter = require("events");
+
+// ─── Filter Lists ───────────────────────────────────────────────────────────
+
+const API_RESOURCE_TYPES = new Set([
+  "XHR", "Fetch", "WebSocket", "Other",
+]);
+
+const IGNORED_DOMAINS = [
+  "google-analytics.com",
+  "googletagmanager.com",
+  "doubleclick.net",
+  "facebook.net",
+  "hotjar.com",
+  "clarity.ms",
+  "cdn.jsdelivr.net",
+  "fonts.googleapis.com",
+  "fonts.gstatic.com",
+  "firebaseinstallations.googleapis.com",
+  "firebase.googleapis.com",
+  "firebaselogging-pa.googleapis.com",
+];
+
+const IGNORED_EXTENSIONS = new Set([
+  ".png", ".jpg", ".jpeg", ".gif", ".svg",
+  ".woff", ".woff2", ".ttf", ".eot",
+  ".ico", ".mp4", ".webp", ".webm",
+  ".css", ".js", ".map", ".json5",
+]);
+
+const ALWAYS_CAPTURE_PATTERNS = [
+  "/api/", "/v1/", "/v2/", "/v3/",
+  "/graphql", "/rest/", "/rpc/",
+];
+
+const ALWAYS_CAPTURE_DOMAINS = [
+  "onrender.com", "herokuapp.com", "vercel.app",
+  "railway.app", "supabase.co", "netlify.app",
+  "netlify.com", "render.com", "vercel.com",
+  "firestore.googleapis.com",
+  "identitytoolkit.googleapis.com",
+  "securetoken.googleapis.com",
+];
+
+// ─── Helpers ────────────────────────────────────────────────────────────────
+
+function isIgnoredDomain(url) {
+  if (ALWAYS_CAPTURE_DOMAINS.some((d) => url.includes(d))) return false;
+  return IGNORED_DOMAINS.some((d) => url.includes(d));
+}
+
+function isIgnoredExtension(url) {
+  try {
+    const pathname = new URL(url).pathname.toLowerCase();
+    return [...IGNORED_EXTENSIONS].some((ext) => pathname.endsWith(ext));
+  } catch {
+    return false;
+  }
+}
+
+function isApiRequest(type, url) {
+  if (!API_RESOURCE_TYPES.has(type)) return false;
+  if (isIgnoredDomain(url)) return false;
+  if (isIgnoredExtension(url)) return false;
+  return true;
+}
+
+// ─── NetworkCollector Class ─────────────────────────────────────────────────
+
+class NetworkCollector extends EventEmitter {
+  /**
+   * @param {import('puppeteer-core').Page} page
+   * @param {object} options
+   * @param {number} [options.responsePreviewLimit=10000]
+   */
+  constructor(page, options = {}) {
+    super();
+    this.page = page;
+    this.cdpClient = null;
+    this.capturedMap = new Map();
+    this.idCounter = 0;
+    this.responsePreviewLimit = options.responsePreviewLimit || 10000;
+    this._active = false;
+  }
+
+  /** Start capturing network traffic via CDP. */
+  async start() {
+    this.cdpClient = await this.page.createCDPSession();
+    await this.cdpClient.send("Network.enable");
+    this._active = true;
+
+    // ── Request initiated ──
+    this.cdpClient.on("Network.requestWillBeSent", (event) => {
+      if (!this._active) return;
+      const { requestId, request, type } = event;
+      if (!isApiRequest(type, request.url)) return;
+
+      this.idCounter++;
+      const entry = {
+        id: this.idCounter,
+        url: request.url,
+        method: request.method,
+        requestHeaders: request.headers,
+        payload: request.postData || null,
+        type: (type || "other").toLowerCase(),
+        status: null,
+        responseHeaders: null,
+        response: null,
+        timestamp: Date.now(),
+        _requestId: requestId,
+      };
+      this.capturedMap.set(requestId, entry);
+      this.emit("request", this._clean(entry));
+    });
+
+    // ── Response headers received ──
+    this.cdpClient.on("Network.responseReceived", (event) => {
+      if (!this._active) return;
+      const { requestId, response } = event;
+      const entry = this.capturedMap.get(requestId);
+      if (!entry) return;
+
+      entry.status = response.status;
+      entry.responseHeaders = response.headers;
+      this.emit("response", this._clean(entry));
+    });
+
+    // ── Response body available ──
+    this.cdpClient.on("Network.loadingFinished", async (event) => {
+      if (!this._active) return;
+      const { requestId } = event;
+      const entry = this.capturedMap.get(requestId);
+      if (!entry) return;
+
+      try {
+        const { body, base64Encoded } = await this.cdpClient.send(
+          "Network.getResponseBody",
+          { requestId }
+        );
+        entry.response = base64Encoded
+          ? "[Binary data]"
+          : body.slice(0, this.responsePreviewLimit);
+      } catch {
+        entry.response = "[Could not read response body]";
+      }
+      this.emit("complete", this._clean(entry));
+    });
+
+    // ── Request failed ──
+    this.cdpClient.on("Network.loadingFailed", (event) => {
+      if (!this._active) return;
+      const { requestId, errorText } = event;
+      const entry = this.capturedMap.get(requestId);
+      if (!entry) return;
+      entry.response = `[Request failed: ${errorText}]`;
+      this.emit("failed", this._clean(entry));
+    });
+  }
+
+  /** Stop capturing and detach CDP session. */
+  async stop() {
+    this._active = false;
+    if (this.cdpClient) {
+      try { await this.cdpClient.detach(); } catch { /* already detached */ }
+      this.cdpClient = null;
+    }
+  }
+
+  /** Number of captured API calls. */
+  get count() {
+    return this.capturedMap.size;
+  }
+
+  /**
+   * Return all captured requests as a sorted array (cleaned of internals).
+   */
+  getResults() {
+    return Array.from(this.capturedMap.values())
+      .map((e) => this._clean(e))
+      .sort((a, b) => a.id - b.id);
+  }
+
+  /**
+   * Wait until no new API requests arrive for `silenceMs`,
+   * or until `maxWaitMs` has elapsed — whichever comes first.
+   */
+  waitForSilence(silenceMs = 3000, maxWaitMs = 20000) {
+    return new Promise((resolve) => {
+      let silenceTimer = null;
+      let maxTimer = null;
+
+      const done = () => {
+        if (silenceTimer) clearTimeout(silenceTimer);
+        if (maxTimer) clearTimeout(maxTimer);
+        this.removeListener("request", resetSilence);
+        resolve();
+      };
+
+      const resetSilence = () => {
+        if (silenceTimer) clearTimeout(silenceTimer);
+        silenceTimer = setTimeout(done, silenceMs);
+      };
+
+      this.on("request", resetSilence);
+      maxTimer = setTimeout(done, maxWaitMs);
+      resetSilence(); // start initial silence timer
+    });
+  }
+
+  /**
+   * Wait for at least `count` *new* requests to arrive (from now),
+   * or until `timeoutMs` elapses.
+   */
+  waitForNewRequests(count = 1, timeoutMs = 10000) {
+    return new Promise((resolve) => {
+      const startCount = this.count;
+      let timer = null;
+
+      const check = () => {
+        if (this.count - startCount >= count) {
+          clearTimeout(timer);
+          this.removeListener("request", check);
+          resolve(true);
+        }
+      };
+
+      this.on("request", check);
+      timer = setTimeout(() => {
+        this.removeListener("request", check);
+        resolve(false);
+      }, timeoutMs);
+    });
+  }
+
+  /** Strip internal fields before exposing to consumers. */
+  _clean(entry) {
+    const { _requestId, timestamp, ...rest } = entry;
+    return rest;
+  }
+}
+
+module.exports = { NetworkCollector };

package/puppeteer/sessionStore.js

@@ -0,0 +1,140 @@
+/**
+ * SessionStore — Manages active Puppeteer sessions for session-aware replay.
+ *
+ * Each scan keeps its browser/page alive for a TTL window so replay
+ * requests can execute inside the same authenticated browser context
+ * (cookies, tokens, service-worker caches, IndexedDB auth state).
+ *
+ * Sessions auto-expire after SESSION_TTL_MS to avoid resource leaks.
+ */
+
+const crypto = require("crypto");
+
+const SESSION_TTL_MS = 10 * 60 * 1000; // 10 minutes
+
+class SessionStore {
+  constructor() {
+    /** @type {Map<string, { engine: SessionEngine, createdAt: number, timer: NodeJS.Timeout }>} */
+    this._sessions = new Map();
+  }
+
+  /**
+   * Store a SessionEngine reference keyed by a new scanId.
+   * Returns the generated scanId.
+   *
+   * @param {import('./interceptor').SessionEngine} engine
+   * @returns {string} scanId
+   */
+  store(engine) {
+    const scanId = crypto.randomUUID();
+
+    // Auto-expire timer
+    const timer = setTimeout(() => {
+      this.destroy(scanId);
+    }, SESSION_TTL_MS);
+
+    this._sessions.set(scanId, {
+      engine,
+      createdAt: Date.now(),
+      timer,
+    });
+
+    console.log(`[SessionStore] Stored session ${scanId} (TTL ${SESSION_TTL_MS / 1000}s)`);
+    return scanId;
+  }
+
+  /**
+   * Retrieve a session engine by scanId.
+   * Returns null if not found or expired.
+   *
+   * @param {string} scanId
+   * @returns {import('./interceptor').SessionEngine | null}
+   */
+  get(scanId) {
+    const entry = this._sessions.get(scanId);
+    if (!entry) return null;
+    return entry.engine;
+  }
+
+  /**
+   * Check if a session exists and is alive.
+   * @param {string} scanId
+   * @returns {boolean}
+   */
+  has(scanId) {
+    return this._sessions.has(scanId);
+  }
+
+  /**
+   * Extend the TTL of a session (called after successful replay).
+   * @param {string} scanId
+   */
+  touch(scanId) {
+    const entry = this._sessions.get(scanId);
+    if (!entry) return;
+
+    clearTimeout(entry.timer);
+    entry.timer = setTimeout(() => {
+      this.destroy(scanId);
+    }, SESSION_TTL_MS);
+  }
+
+  /**
+   * Destroy a session: close browser, remove from store.
+   * @param {string} scanId
+   */
+  async destroy(scanId) {
+    const entry = this._sessions.get(scanId);
+    if (!entry) return;
+
+    clearTimeout(entry.timer);
+    this._sessions.delete(scanId);
+
+    try {
+      if (entry.engine && entry.engine.browser) {
+        console.log(`[SessionStore] Closing session ${scanId}`);
+        await entry.engine.browser.close();
+        entry.engine.browser = null;
+        entry.engine.page = null;
+        entry.engine._running = false;
+      }
+    } catch (err) {
+      console.warn(`[SessionStore] Error closing session ${scanId}:`, err.message);
+    }
+  }
+
+  /**
+   * Destroy all sessions.
+   */
+  async destroyAll() {
+    const ids = [...this._sessions.keys()];
+    for (const id of ids) {
+      await this.destroy(id);
+    }
+  }
+
+  /**
+   * Get info about all active sessions.
+   */
+  list() {
+    const sessions = [];
+    for (const [id, entry] of this._sessions) {
+      sessions.push({
+        scanId: id,
+        createdAt: entry.createdAt,
+        alive: !!(entry.engine && entry.engine.page),
+        ageMs: Date.now() - entry.createdAt,
+      });
+    }
+    return sessions;
+  }
+
+  get size() {
+    return this._sessions.size;
+  }
+}
+
+// Singleton instance
+const sessionStore = new SessionStore();
+
+module.exports = { sessionStore, SESSION_TTL_MS };