@vulcn/driver-browser 0.1.1 → 0.2.0

package/dist/index.cjs

@@ -26,6 +26,7 @@ __export(index_exports, {
   BrowserStepSchema: () => BrowserStepSchema,
   checkBrowsers: () => checkBrowsers,
   configSchema: () => configSchema,
+  crawlAndBuildSessions: () => crawlAndBuildSessions,
   default: () => index_default,
   installBrowsers: () => installBrowsers,
   launchBrowser: () => launchBrowser
@@ -434,6 +435,7 @@ var BrowserRunner = class _BrowserRunner {
     });
     const context = await browser.newContext({ viewport });
     const page = await context.newPage();
+    await ctx.options.onPageReady?.(page);
     const eventFindings = [];
     let currentPayloadInfo = null;
     const dialogHandler = async (dialog) => {
@@ -491,13 +493,24 @@ var BrowserRunner = class _BrowserRunner {
         (step) => step.type === "browser.input" && step.injectable !== false
       );
       const allPayloads = [];
-      for (const payloadSet of payloads) {
-        for (const value of payloadSet.payloads) {
-          allPayloads.push({ payloadSet, value });
+      const payloadsByCategory = payloads.map(
+        (ps) => ps.payloads.map((value) => ({ payloadSet: ps, value }))
+      );
+      const maxLen = Math.max(...payloadsByCategory.map((c) => c.length));
+      for (let i = 0; i < maxLen; i++) {
+        for (const category of payloadsByCategory) {
+          if (i < category.length) {
+            allPayloads.push(category[i]);
+          }
         }
       }
+      const confirmedTypes = /* @__PURE__ */ new Set();
       for (const injectableStep of injectableSteps) {
         for (const { payloadSet, value } of allPayloads) {
+          const stepTypeKey = `${injectableStep.id}::${payloadSet.category}`;
+          if (confirmedTypes.has(stepTypeKey)) {
+            continue;
+          }
           try {
             currentPayloadInfo = {
               stepId: injectableStep.id,
@@ -521,8 +534,16 @@ var BrowserRunner = class _BrowserRunner {
             if (reflectionFinding) {
               allFindings.push(reflectionFinding);
             }
+            const seenKeys = /* @__PURE__ */ new Set();
             for (const finding of allFindings) {
-              ctx.addFinding(finding);
+              const dedupKey = `${finding.type}::${finding.stepId}::${finding.title}`;
+              if (!seenKeys.has(dedupKey)) {
+                seenKeys.add(dedupKey);
+                ctx.addFinding(finding);
+              }
+            }
+            if (allFindings.length > 0) {
+              confirmedTypes.add(stepTypeKey);
             }
             eventFindings.length = 0;
             payloadsTested++;
@@ -536,6 +557,7 @@ var BrowserRunner = class _BrowserRunner {
       page.off("dialog", dialogHandler);
       page.off("console", consoleHandler);
       currentPayloadInfo = null;
+      await ctx.options.onBeforeClose?.(page);
       await browser.close();
     }
     return {
@@ -677,6 +699,356 @@ var BrowserRunner = class _BrowserRunner {
   }
 };
 
+// src/crawler.ts
+var INJECTABLE_INPUT_TYPES = /* @__PURE__ */ new Set([
+  "text",
+  "search",
+  "url",
+  "email",
+  "tel",
+  "password",
+  "textarea",
+  ""
+]);
+var CRAWL_DEFAULTS = {
+  maxDepth: 2,
+  maxPages: 20,
+  pageTimeout: 1e4,
+  sameOrigin: true
+};
+async function crawlAndBuildSessions(config, options = {}) {
+  const opts = { ...CRAWL_DEFAULTS, ...options };
+  const startUrl = config.startUrl;
+  let normalizedUrl;
+  try {
+    normalizedUrl = new URL(startUrl);
+  } catch {
+    throw new Error(`Invalid URL: ${startUrl}`);
+  }
+  const origin = normalizedUrl.origin;
+  const visited = /* @__PURE__ */ new Set();
+  const allForms = [];
+  const queue = [[normalizedUrl.href, 0]];
+  const { browser } = await launchBrowser({
+    browser: config.browser ?? "chromium",
+    headless: config.headless ?? true
+  });
+  const context = await browser.newContext({
+    viewport: config.viewport ?? { width: 1280, height: 720 }
+  });
+  try {
+    while (queue.length > 0 && visited.size < opts.maxPages) {
+      const [url, depth] = queue.shift();
+      const normalizedPageUrl = normalizeUrl(url);
+      if (visited.has(normalizedPageUrl)) continue;
+      visited.add(normalizedPageUrl);
+      console.log(`[crawler] [depth=${depth}] Crawling: ${normalizedPageUrl}`);
+      const page = await context.newPage();
+      try {
+        await page.goto(normalizedPageUrl, {
+          waitUntil: "domcontentloaded",
+          timeout: opts.pageTimeout
+        });
+        await page.waitForTimeout(1e3);
+        const forms = await discoverForms(page, normalizedPageUrl);
+        allForms.push(...forms);
+        const injectableCount = forms.reduce(
+          (s, f) => s + f.inputs.filter((i) => i.injectable).length,
+          0
+        );
+        console.log(
+          `[crawler]   Found ${forms.length} form(s), ${injectableCount} injectable input(s)`
+        );
+        opts.onPageCrawled?.(normalizedPageUrl, forms.length);
+        if (depth < opts.maxDepth) {
+          const links = await discoverLinks(page, origin, opts.sameOrigin);
+          for (const link of links) {
+            const normalizedLink = normalizeUrl(link);
+            if (!visited.has(normalizedLink)) {
+              queue.push([normalizedLink, depth + 1]);
+            }
+          }
+          console.log(`[crawler]   Found ${links.length} link(s) to follow`);
+        }
+      } catch (err) {
+        console.warn(
+          `[crawler]   Failed: ${err instanceof Error ? err.message : String(err)}`
+        );
+      } finally {
+        await page.close();
+      }
+    }
+  } finally {
+    await browser.close();
+  }
+  console.log(
+    `[crawler] Complete: ${visited.size} page(s), ${allForms.length} form(s)`
+  );
+  return buildSessions(allForms);
+}
+async function discoverForms(page, pageUrl) {
+  const forms = [];
+  const explicitForms = await page.evaluate(() => {
+    const results = [];
+    const formElements = document.querySelectorAll("form");
+    formElements.forEach((form, formIndex) => {
+      const inputs = [];
+      const inputEls = form.querySelectorAll(
+        'input, textarea, [contenteditable="true"]'
+      );
+      inputEls.forEach((input, inputIndex) => {
+        const el = input;
+        const type = el.tagName.toLowerCase() === "textarea" ? "textarea" : el.getAttribute("type") || "text";
+        const name = el.name || el.id || `input-${inputIndex}`;
+        let selector = "";
+        if (el.id) {
+          selector = `#${CSS.escape(el.id)}`;
+        } else if (el.name) {
+          selector = `form:nth-of-type(${formIndex + 1}) [name="${CSS.escape(el.name)}"]`;
+        } else {
+          selector = `form:nth-of-type(${formIndex + 1}) ${el.tagName.toLowerCase()}:nth-of-type(${inputIndex + 1})`;
+        }
+        inputs.push({
+          selector,
+          type,
+          name,
+          placeholder: el.placeholder || ""
+        });
+      });
+      let submitSelector = null;
+      const submitBtn = form.querySelector('button[type="submit"], input[type="submit"]') || form.querySelector("button:not([type])") || form.querySelector('button, input[type="button"]');
+      if (submitBtn) {
+        const btn = submitBtn;
+        if (btn.id) {
+          submitSelector = `#${CSS.escape(btn.id)}`;
+        } else {
+          const tag = btn.tagName.toLowerCase();
+          const type = btn.getAttribute("type");
+          if (type) {
+            submitSelector = `form:nth-of-type(${formIndex + 1}) ${tag}[type="${type}"]`;
+          } else {
+            submitSelector = `form:nth-of-type(${formIndex + 1}) ${tag}`;
+          }
+        }
+      }
+      results.push({
+        formIndex,
+        action: form.action || "",
+        method: (form.method || "GET").toUpperCase(),
+        inputs,
+        submitSelector
+      });
+    });
+    return results;
+  });
+  for (const form of explicitForms) {
+    if (form.inputs.length === 0) continue;
+    forms.push({
+      pageUrl,
+      formSelector: `form:nth-of-type(${form.formIndex + 1})`,
+      action: form.action,
+      method: form.method,
+      inputs: form.inputs.map((input) => ({
+        selector: input.selector,
+        type: input.type,
+        name: input.name,
+        injectable: INJECTABLE_INPUT_TYPES.has(input.type.toLowerCase()),
+        placeholder: input.placeholder || void 0
+      })),
+      submitSelector: form.submitSelector
+    });
+  }
+  const standaloneInputs = await page.evaluate(() => {
+    const results = [];
+    const allInputs = document.querySelectorAll(
+      'input:not(form input), textarea:not(form textarea), [contenteditable="true"]:not(form [contenteditable])'
+    );
+    allInputs.forEach((input) => {
+      const el = input;
+      const type = el.tagName.toLowerCase() === "textarea" ? "textarea" : el.getAttribute("type") || "text";
+      const name = el.name || el.id || "";
+      let selector = "";
+      if (el.id) {
+        selector = `#${CSS.escape(el.id)}`;
+      } else if (el.name) {
+        selector = `[name="${CSS.escape(el.name)}"]`;
+      } else {
+        selector = `${el.tagName.toLowerCase()}[type="${type}"]`;
+      }
+      let nearbyButtonSelector = null;
+      const parent = el.parentElement;
+      if (parent) {
+        const btn = parent.querySelector("button") || parent.querySelector('input[type="submit"]') || parent.querySelector('input[type="button"]');
+        if (btn) {
+          const btnEl = btn;
+          if (btnEl.id) {
+            nearbyButtonSelector = `#${CSS.escape(btnEl.id)}`;
+          }
+        }
+      }
+      results.push({
+        selector,
+        type,
+        name,
+        placeholder: el.placeholder || "",
+        nearbyButtonSelector
+      });
+    });
+    return results;
+  });
+  for (const input of standaloneInputs) {
+    if (!INJECTABLE_INPUT_TYPES.has(input.type.toLowerCase())) continue;
+    forms.push({
+      pageUrl,
+      formSelector: "(standalone)",
+      action: pageUrl,
+      method: "GET",
+      inputs: [
+        {
+          selector: input.selector,
+          type: input.type,
+          name: input.name,
+          injectable: true,
+          placeholder: input.placeholder || void 0
+        }
+      ],
+      submitSelector: input.nearbyButtonSelector
+    });
+  }
+  return forms;
+}
+var REDIRECT_PARAMS = /* @__PURE__ */ new Set([
+  "to",
+  "url",
+  "redirect",
+  "redirect_uri",
+  "redirect_url",
+  "return",
+  "return_url",
+  "returnto",
+  "next",
+  "goto",
+  "dest",
+  "destination",
+  "continue",
+  "target",
+  "rurl",
+  "out",
+  "link",
+  "forward"
+]);
+function isExternalRedirectLink(link, origin) {
+  try {
+    const parsed = new URL(link);
+    if (parsed.origin !== origin) return false;
+    for (const [key, value] of parsed.searchParams) {
+      if (REDIRECT_PARAMS.has(key.toLowerCase())) {
+        try {
+          const targetUrl = new URL(value);
+          if (targetUrl.origin !== origin) return true;
+        } catch {
+        }
+      }
+    }
+    return false;
+  } catch {
+    return false;
+  }
+}
+async function discoverLinks(page, origin, sameOrigin) {
+  const links = await page.evaluate(() => {
+    return Array.from(document.querySelectorAll("a[href]")).map((a) => a.href).filter((href) => href.startsWith("http"));
+  });
+  return links.filter((link) => {
+    try {
+      const linkOrigin = new URL(link).origin;
+      if (sameOrigin && linkOrigin !== origin) return false;
+      if (isExternalRedirectLink(link, origin)) return false;
+      return true;
+    } catch {
+      return false;
+    }
+  });
+}
+function buildSessions(forms) {
+  const targetForms = forms.filter((f) => f.inputs.some((i) => i.injectable));
+  return targetForms.map((form, idx) => buildSessionForForm(form, idx));
+}
+function buildSessionForForm(form, index) {
+  const steps = [];
+  let stepNum = 1;
+  steps.push({
+    id: `step-${stepNum++}`,
+    type: "browser.navigate",
+    url: form.pageUrl,
+    timestamp: Date.now()
+  });
+  const injectableInputs = form.inputs.filter((i) => i.injectable);
+  for (const input of injectableInputs) {
+    steps.push({
+      id: `step-${stepNum++}`,
+      type: "browser.input",
+      selector: input.selector,
+      value: "test",
+      injectable: true,
+      timestamp: Date.now() + stepNum * 100
+    });
+  }
+  if (form.submitSelector) {
+    steps.push({
+      id: `step-${stepNum++}`,
+      type: "browser.click",
+      selector: form.submitSelector,
+      timestamp: Date.now() + stepNum * 100
+    });
+  } else {
+    steps.push({
+      id: `step-${stepNum++}`,
+      type: "browser.keypress",
+      key: "Enter",
+      timestamp: Date.now() + stepNum * 100
+    });
+  }
+  const inputNames = injectableInputs.map((i) => i.name || i.type).join(", ");
+  const pagePath = (() => {
+    try {
+      return new URL(form.pageUrl).pathname;
+    } catch {
+      return form.pageUrl;
+    }
+  })();
+  return {
+    name: `Crawl: ${pagePath} \u2014 form ${index + 1} (${inputNames})`,
+    driver: "browser",
+    driverConfig: {
+      startUrl: form.pageUrl,
+      browser: "chromium",
+      headless: true,
+      viewport: { width: 1280, height: 720 }
+    },
+    steps,
+    metadata: {
+      recordedAt: (/* @__PURE__ */ new Date()).toISOString(),
+      version: "0.3.0",
+      source: "crawler",
+      formAction: form.action,
+      formMethod: form.method
+    }
+  };
+}
+function normalizeUrl(url) {
+  try {
+    const parsed = new URL(url);
+    parsed.hash = "";
+    if (parsed.pathname !== "/" && parsed.pathname.endsWith("/")) {
+      parsed.pathname = parsed.pathname.slice(0, -1);
+    }
+    return parsed.href;
+  } catch {
+    return url;
+  }
+}
+
 // src/index.ts
 var configSchema = import_zod.z.object({
   /** Starting URL for recording */
@@ -746,6 +1118,18 @@ var recorderDriver = {
   async start(config, options) {
     const parsedConfig = configSchema.parse(config);
     return BrowserRecorder.start(parsedConfig, options);
+  },
+  async crawl(config, options) {
+    const parsedConfig = configSchema.parse(config);
+    return crawlAndBuildSessions(
+      {
+        startUrl: parsedConfig.startUrl ?? "",
+        browser: parsedConfig.browser,
+        headless: parsedConfig.headless,
+        viewport: parsedConfig.viewport
+      },
+      options
+    );
   }
 };
 var runnerDriver = {
@@ -772,6 +1156,7 @@ var index_default = browserDriver;
   BrowserStepSchema,
   checkBrowsers,
   configSchema,
+  crawlAndBuildSessions,
   installBrowsers,
   launchBrowser
 });