@detiq/crawler 0.1.3

package/dist/form-detector.js

@@ -0,0 +1,58 @@
+function guessPurpose(fields, action) {
+  const names = fields.map((f) => f.name?.toLowerCase() + (f.label ?? "").toLowerCase()).join(" ");
+  if (/password/.test(names) && /email|username|user/.test(names) && fields.length <= 4) return "login";
+  if (/password/.test(names) && /confirm|repeat/.test(names)) return "registration";
+  if (/search|query|q\b/.test(names) && fields.length <= 3) return "search";
+  if (/card|payment|cvv|expir/.test(names)) return "checkout";
+  if (/message|subject|contact/.test(names)) return "contact";
+  if (/setting|preference|profile/.test(names)) return "settings";
+  return "unknown";
+}
+async function detectForms(page) {
+  try {
+    return await page.$$eval("form", (forms) => forms.map((form, index) => {
+      const inputs = Array.from(form.querySelectorAll('input,select,textarea,button[type="submit"]'));
+      const fields = [];
+      for (const input of inputs) {
+        const tag = input.tagName.toLowerCase();
+        const type = tag === "button" ? "submit" : input.type || tag;
+        if (["submit", "reset", "button", "hidden"].includes(type)) continue;
+        let label = "";
+        if (input.id) {
+          const lbl = form.querySelector(`label[for="${input.id}"]`);
+          if (lbl) label = (lbl.textContent ?? "").trim();
+        }
+        if (!label) {
+          const parent = input.closest("label");
+          if (parent) label = (parent.textContent ?? "").replace(input.value ?? "", "").trim().slice(0, 80);
+        }
+        const options = tag === "select" ? Array.from(input.options ?? []).map((o) => o.text?.trim()).filter(Boolean).slice(0, 20) : tag === "input" && type === "radio" ? [input.value] : void 0;
+        fields.push({
+          name: input.name || input.id || "",
+          type,
+          label: label || void 0,
+          required: !!input.required,
+          placeholder: input.placeholder || void 0,
+          minLength: input.minLength > 0 ? input.minLength : void 0,
+          maxLength: input.maxLength > 0 && input.maxLength < 524288 ? input.maxLength : void 0,
+          min: input.min || void 0,
+          max: input.max || void 0,
+          pattern: input.pattern || void 0,
+          options,
+          autocomplete: input.autocomplete || void 0
+        });
+      }
+      const submitBtn = form.querySelector('button[type="submit"],input[type="submit"]');
+      const hasFileUpload = !!form.querySelector('input[type="file"]');
+      const action = form.action || void 0;
+      const method = (form.method || "get").toUpperCase();
+      return { index, action, method, fields, submitText: submitBtn?.textContent?.trim() || submitBtn?.value || void 0, hasFileUpload };
+    })).then((forms) => forms.map((f) => ({ ...f, purpose: guessPurpose(f.fields, f.action) })));
+  } catch {
+    return [];
+  }
+}
+export {
+  detectForms
+};
+//# sourceMappingURL=form-detector.js.map

package/dist/form-detector.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/form-detector.ts"],
+  "sourcesContent": ["/**\n * Form detector \u2014 extract form metadata from crawled pages.\n *\n * For each form found on a page, captures: action URL, method, all input fields\n * with their types/names/required/validation constraints, and submit button text.\n *\n * This data feeds ZeTa's test generator to create form validation test cases\n * automatically (happy path + boundary/error cases per field type).\n */\n\nimport type { Page } from 'playwright';\n\nexport interface FormField {\n  name: string;\n  type: string;        // text, email, password, number, select, textarea, checkbox, radio, file, date\n  label?: string;\n  required: boolean;\n  placeholder?: string;\n  minLength?: number;\n  maxLength?: number;\n  min?: string;\n  max?: string;\n  pattern?: string;\n  options?: string[]; // for select/radio\n  autocomplete?: string;\n}\n\nexport interface DetectedForm {\n  index: number;\n  action?: string;\n  method: string;\n  fields: FormField[];\n  submitText?: string;\n  hasFileUpload: boolean;\n  purpose: 'login' | 'search' | 'registration' | 'contact' | 'checkout' | 'settings' | 'unknown';\n}\n\nfunction guessPurpose(fields: FormField[], action?: string): DetectedForm['purpose'] {\n  const names = fields.map(f => f.name?.toLowerCase() + (f.label ?? '').toLowerCase()).join(' ');\n  if (/password/.test(names) && /email|username|user/.test(names) && fields.length <= 4) return 'login';\n  if (/password/.test(names) && /confirm|repeat/.test(names)) return 'registration';\n  if (/search|query|q\\b/.test(names) && fields.length <= 3) return 'search';\n  if (/card|payment|cvv|expir/.test(names)) return 'checkout';\n  if (/message|subject|contact/.test(names)) return 'contact';\n  if (/setting|preference|profile/.test(names)) return 'settings';\n  return 'unknown';\n}\n\nexport async function detectForms(page: Page): Promise<DetectedForm[]> {\n  try {\n    return await page.$$eval('form', (forms) => forms.map((form: any, index: number) => {\n      const inputs = Array.from(form.querySelectorAll('input,select,textarea,button[type=\"submit\"]'));\n      const fields: any[] = [];\n\n      for (const input of inputs as any[]) {\n        const tag = input.tagName.toLowerCase();\n        const type = tag === 'button' ? 'submit' : (input.type || tag);\n        if (['submit', 'reset', 'button', 'hidden'].includes(type)) continue;\n\n        // Find associated label\n        let label = '';\n        if (input.id) {\n          const lbl = form.querySelector(`label[for=\"${input.id}\"]`);\n          if (lbl) label = (lbl.textContent ?? '').trim();\n        }\n        if (!label) {\n          const parent = input.closest('label');\n          if (parent) label = (parent.textContent ?? '').replace(input.value ?? '', '').trim().slice(0, 80);\n        }\n\n        const options = tag === 'select'\n          ? Array.from(input.options ?? []).map((o: any) => o.text?.trim()).filter(Boolean).slice(0, 20)\n          : tag === 'input' && type === 'radio'\n          ? [input.value]\n          : undefined;\n\n        fields.push({\n          name: input.name || input.id || '',\n          type,\n          label: label || undefined,\n          required: !!input.required,\n          placeholder: input.placeholder || undefined,\n          minLength: input.minLength > 0 ? input.minLength : undefined,\n          maxLength: input.maxLength > 0 && input.maxLength < 524288 ? input.maxLength : undefined,\n          min: input.min || undefined,\n          max: input.max || undefined,\n          pattern: input.pattern || undefined,\n          options,\n          autocomplete: input.autocomplete || undefined,\n        });\n      }\n\n      const submitBtn = form.querySelector('button[type=\"submit\"],input[type=\"submit\"]') as any;\n      const hasFileUpload = !!form.querySelector('input[type=\"file\"]');\n      const action = form.action || undefined;\n      const method = (form.method || 'get').toUpperCase();\n\n      return { index, action, method, fields, submitText: submitBtn?.textContent?.trim() || submitBtn?.value || undefined, hasFileUpload };\n    })).then(forms => forms.map((f: any) => ({ ...f, purpose: guessPurpose(f.fields, f.action) })));\n  } catch {\n    return [];\n  }\n}\n"],
+  "mappings": "AAqCA,SAAS,aAAa,QAAqB,QAA0C;AACnF,QAAM,QAAQ,OAAO,IAAI,OAAK,EAAE,MAAM,YAAY,KAAK,EAAE,SAAS,IAAI,YAAY,CAAC,EAAE,KAAK,GAAG;AAC7F,MAAI,WAAW,KAAK,KAAK,KAAK,sBAAsB,KAAK,KAAK,KAAK,OAAO,UAAU,EAAG,QAAO;AAC9F,MAAI,WAAW,KAAK,KAAK,KAAK,iBAAiB,KAAK,KAAK,EAAG,QAAO;AACnE,MAAI,mBAAmB,KAAK,KAAK,KAAK,OAAO,UAAU,EAAG,QAAO;AACjE,MAAI,yBAAyB,KAAK,KAAK,EAAG,QAAO;AACjD,MAAI,0BAA0B,KAAK,KAAK,EAAG,QAAO;AAClD,MAAI,6BAA6B,KAAK,KAAK,EAAG,QAAO;AACrD,SAAO;AACT;AAEA,eAAsB,YAAY,MAAqC;AACrE,MAAI;AACF,WAAO,MAAM,KAAK,OAAO,QAAQ,CAAC,UAAU,MAAM,IAAI,CAAC,MAAW,UAAkB;AAClF,YAAM,SAAS,MAAM,KAAK,KAAK,iBAAiB,6CAA6C,CAAC;AAC9F,YAAM,SAAgB,CAAC;AAEvB,iBAAW,SAAS,QAAiB;AACnC,cAAM,MAAM,MAAM,QAAQ,YAAY;AACtC,cAAM,OAAO,QAAQ,WAAW,WAAY,MAAM,QAAQ;AAC1D,YAAI,CAAC,UAAU,SAAS,UAAU,QAAQ,EAAE,SAAS,IAAI,EAAG;AAG5D,YAAI,QAAQ;AACZ,YAAI,MAAM,IAAI;AACZ,gBAAM,MAAM,KAAK,cAAc,cAAc,MAAM,EAAE,IAAI;AACzD,cAAI,IAAK,UAAS,IAAI,eAAe,IAAI,KAAK;AAAA,QAChD;AACA,YAAI,CAAC,OAAO;AACV,gBAAM,SAAS,MAAM,QAAQ,OAAO;AACpC,cAAI,OAAQ,UAAS,OAAO,eAAe,IAAI,QAAQ,MAAM,SAAS,IAAI,EAAE,EAAE,KAAK,EAAE,MAAM,GAAG,EAAE;AAAA,QAClG;AAEA,cAAM,UAAU,QAAQ,WACpB,MAAM,KAAK,MAAM,WAAW,CAAC,CAAC,EAAE,IAAI,CAAC,MAAW,EAAE,MAAM,KAAK,CAAC,EAAE,OAAO,OAAO,EAAE,MAAM,GAAG,EAAE,IAC3F,QAAQ,WAAW,SAAS,UAC5B,CAAC,MAAM,KAAK,IACZ;AAEJ,eAAO,KAAK;AAAA,UACV,MAAM,MAAM,QAAQ,MAAM,MAAM;AAAA,UAChC;AAAA,UACA,OAAO,SAAS;AAAA,UAChB,UAAU,CAAC,CAAC,MAAM;AAAA,UAClB,aAAa,MAAM,eAAe;AAAA,UAClC,WAAW,MAAM,YAAY,IAAI,MAAM,YAAY;AAAA,UACnD,WAAW,MAAM,YAAY,KAAK,MAAM,YAAY,SAAS,MAAM,YAAY;AAAA,UAC/E,KAAK,MAAM,OAAO;AAAA,UAClB,KAAK,MAAM,OAAO;AAAA,UAClB,SAAS,MAAM,WAAW;AAAA,UAC1B;AAAA,UACA,cAAc,MAAM,gBAAgB;AAAA,QACtC,CAAC;AAAA,MACH;AAEA,YAAM,YAAY,KAAK,cAAc,4CAA4C;AACjF,YAAM,gBAAgB,CAAC,CAAC,KAAK,cAAc,oBAAoB;AAC/D,YAAM,SAAS,KAAK,UAAU;AAC9B,YAAM,UAAU,KAAK,UAAU,OAAO,YAAY;AAElD,aAAO,EAAE,OAAO,QAAQ,QAAQ,QAAQ,YAAY,WAAW,aAAa,KAAK,KAAK,WAAW,SAAS,QAAW,cAAc;AAAA,IACrI,CAAC,CAAC,EAAE,KAAK,WAAS,MAAM,IAAI,CAAC,OAAY,EAAE,GAAG,GAAG,SAAS,aAAa,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,CAAC;AAAA,EAChG,QAAQ;AACN,WAAO,CAAC;AAAA,EACV;AACF;",
+  "names": []
+}

package/dist/frontier.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Configurable crawl traversal strategy — matches crawl4AI's BFS/DFS/
+ * Best-First deep-crawl strategies (ZeTa previously had a single hardcoded
+ * FIFO queue, i.e. always BFS).
+ *
+ * Array-like on purpose (`length`, `shift()`, `push()`, `filter()`) so it
+ * drops into crawler.ts's existing queue call sites with a rename, not a
+ * rewrite — BFS's `shift()` behavior is byte-for-byte the same as the plain
+ * array it replaces, so BFS (the default) has zero behavior change.
+ */
+export type TraversalStrategy = 'BFS' | 'DFS' | 'BEST_FIRST';
+export declare class CrawlFrontier {
+    private items;
+    private readonly strategy;
+    private readonly keywords;
+    private readonly urlFilter;
+    constructor(strategy: TraversalStrategy, seedUrls: string[], keywords?: string[], urlFilter?: string);
+    get length(): number;
+    /**
+     * P3b: sparse-parent boost — if the parent page had fewer than 5 interactive
+     * elements (thin/canvas page), its child URLs score +1 in BEST_FIRST mode so
+     * the frontier revisits sparse branches before deep-linked content.
+     */
+    push(url: string, pushOpts?: {
+        parentElements?: number;
+    }): void;
+    /** Dequeues the next URL per this frontier's strategy. undefined if empty. */
+    shift(): string | undefined;
+    filter(predicate: (url: string) => boolean): string[];
+    /** Remove queued URLs not matching the regex pattern. Returns count removed. */
+    filterByPattern(pattern: string): number;
+}

package/dist/frontier.js

@@ -0,0 +1,83 @@
+function scoreUrl(url, keywords) {
+  try {
+    const u = new URL(url);
+    const segments = u.pathname.split("/").filter(Boolean);
+    let score = -segments.length;
+    if (keywords.length > 0) {
+      const lower = url.toLowerCase();
+      for (const kw of keywords) {
+        if (kw && lower.includes(kw)) score += 10;
+      }
+    }
+    return score;
+  } catch {
+    return 0;
+  }
+}
+class CrawlFrontier {
+  items = [];
+  strategy;
+  keywords;
+  urlFilter;
+  constructor(strategy, seedUrls, keywords = [], urlFilter) {
+    this.strategy = strategy;
+    this.keywords = keywords.map((k) => k.toLowerCase()).filter(Boolean);
+    this.urlFilter = urlFilter ? (() => {
+      try {
+        return new RegExp(urlFilter, "i");
+      } catch {
+        return null;
+      }
+    })() : null;
+    for (const url of seedUrls) this.push(url);
+  }
+  get length() {
+    return this.items.length;
+  }
+  /**
+   * P3b: sparse-parent boost — if the parent page had fewer than 5 interactive
+   * elements (thin/canvas page), its child URLs score +1 in BEST_FIRST mode so
+   * the frontier revisits sparse branches before deep-linked content.
+   */
+  push(url, pushOpts) {
+    if (this.urlFilter && !this.urlFilter.test(url)) return;
+    const sparseBoost = this.strategy === "BEST_FIRST" && pushOpts?.parentElements !== void 0 && pushOpts.parentElements < 5 ? 1 : 0;
+    this.items.push({ url, score: this.strategy === "BEST_FIRST" ? scoreUrl(url, this.keywords) + sparseBoost : 0 });
+  }
+  /** Dequeues the next URL per this frontier's strategy. undefined if empty. */
+  shift() {
+    if (this.items.length === 0) return void 0;
+    switch (this.strategy) {
+      case "DFS":
+        return this.items.pop().url;
+      case "BEST_FIRST": {
+        let bestIdx = 0;
+        for (let i = 1; i < this.items.length; i++) {
+          if (this.items[i].score > this.items[bestIdx].score) bestIdx = i;
+        }
+        return this.items.splice(bestIdx, 1)[0].url;
+      }
+      case "BFS":
+      default:
+        return this.items.shift().url;
+    }
+  }
+  filter(predicate) {
+    return this.items.filter((e) => predicate(e.url)).map((e) => e.url);
+  }
+  /** Remove queued URLs not matching the regex pattern. Returns count removed. */
+  filterByPattern(pattern) {
+    try {
+      const re = new RegExp(pattern, "i");
+      const before = this.items.length;
+      this.items = this.items.filter((e) => re.test(e.url));
+      return before - this.items.length;
+    } catch {
+      return 0;
+    }
+  }
+}
+export {
+  CrawlFrontier
+};
+//# sourceMappingURL=frontier.js.map

package/dist/frontier.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/frontier.ts"],
+  "sourcesContent": ["/**\n * Configurable crawl traversal strategy \u2014 matches crawl4AI's BFS/DFS/\n * Best-First deep-crawl strategies (ZeTa previously had a single hardcoded\n * FIFO queue, i.e. always BFS).\n *\n * Array-like on purpose (`length`, `shift()`, `push()`, `filter()`) so it\n * drops into crawler.ts's existing queue call sites with a rename, not a\n * rewrite \u2014 BFS's `shift()` behavior is byte-for-byte the same as the plain\n * array it replaces, so BFS (the default) has zero behavior change.\n */\n\nexport type TraversalStrategy = 'BFS' | 'DFS' | 'BEST_FIRST';\n\ninterface FrontierEntry {\n  url: string;\n  score: number;\n}\n\n/**\n * Best-First scoring heuristic: shallower paths score higher (breadth-like\n * bias toward top-level sections before deep sub-pages), boosted further by\n * any user-supplied keyword appearing in the URL. Simple and transparent by\n * design \u2014 crawl4AI's own default scorer is a similarly simple keyword-\n * relevance heuristic, not a learned model.\n */\nfunction scoreUrl(url: string, keywords: string[]): number {\n  try {\n    const u = new URL(url);\n    const segments = u.pathname.split('/').filter(Boolean);\n    let score = -segments.length;\n    if (keywords.length > 0) {\n      const lower = url.toLowerCase();\n      for (const kw of keywords) {\n        if (kw && lower.includes(kw)) score += 10;\n      }\n    }\n    return score;\n  } catch {\n    return 0;\n  }\n}\n\nexport class CrawlFrontier {\n  private items: FrontierEntry[] = [];\n  private readonly strategy: TraversalStrategy;\n  private readonly keywords: string[];\n  private readonly urlFilter: RegExp | null;\n\n  constructor(strategy: TraversalStrategy, seedUrls: string[], keywords: string[] = [], urlFilter?: string) {\n    this.strategy = strategy;\n    this.keywords = keywords.map((k) => k.toLowerCase()).filter(Boolean);\n    this.urlFilter = urlFilter\n      ? (() => { try { return new RegExp(urlFilter, 'i'); } catch { return null; } })()\n      : null;\n    for (const url of seedUrls) this.push(url);\n  }\n\n  get length(): number {\n    return this.items.length;\n  }\n\n  /**\n   * P3b: sparse-parent boost \u2014 if the parent page had fewer than 5 interactive\n   * elements (thin/canvas page), its child URLs score +1 in BEST_FIRST mode so\n   * the frontier revisits sparse branches before deep-linked content.\n   */\n  push(url: string, pushOpts?: { parentElements?: number }): void {\n    if (this.urlFilter && !this.urlFilter.test(url)) return;\n    const sparseBoost =\n      this.strategy === 'BEST_FIRST' &&\n      pushOpts?.parentElements !== undefined &&\n      pushOpts.parentElements < 5\n        ? 1\n        : 0;\n    this.items.push({ url, score: this.strategy === 'BEST_FIRST' ? scoreUrl(url, this.keywords) + sparseBoost : 0 });\n  }\n\n  /** Dequeues the next URL per this frontier's strategy. undefined if empty. */\n  shift(): string | undefined {\n    if (this.items.length === 0) return undefined;\n    switch (this.strategy) {\n      case 'DFS':\n        return this.items.pop()!.url;\n      case 'BEST_FIRST': {\n        let bestIdx = 0;\n        for (let i = 1; i < this.items.length; i++) {\n          if (this.items[i].score > this.items[bestIdx].score) bestIdx = i;\n        }\n        return this.items.splice(bestIdx, 1)[0].url;\n      }\n      case 'BFS':\n      default:\n        return this.items.shift()!.url;\n    }\n  }\n\n  filter(predicate: (url: string) => boolean): string[] {\n    return this.items.filter((e) => predicate(e.url)).map((e) => e.url);\n  }\n\n  /** Remove queued URLs not matching the regex pattern. Returns count removed. */\n  filterByPattern(pattern: string): number {\n    try {\n      const re = new RegExp(pattern, 'i');\n      const before = this.items.length;\n      this.items = this.items.filter((e) => re.test(e.url));\n      return before - this.items.length;\n    } catch {\n      return 0;\n    }\n  }\n}\n"],
+  "mappings": "AAyBA,SAAS,SAAS,KAAa,UAA4B;AACzD,MAAI;AACF,UAAM,IAAI,IAAI,IAAI,GAAG;AACrB,UAAM,WAAW,EAAE,SAAS,MAAM,GAAG,EAAE,OAAO,OAAO;AACrD,QAAI,QAAQ,CAAC,SAAS;AACtB,QAAI,SAAS,SAAS,GAAG;AACvB,YAAM,QAAQ,IAAI,YAAY;AAC9B,iBAAW,MAAM,UAAU;AACzB,YAAI,MAAM,MAAM,SAAS,EAAE,EAAG,UAAS;AAAA,MACzC;AAAA,IACF;AACA,WAAO;AAAA,EACT,QAAQ;AACN,WAAO;AAAA,EACT;AACF;AAEO,MAAM,cAAc;AAAA,EACjB,QAAyB,CAAC;AAAA,EACjB;AAAA,EACA;AAAA,EACA;AAAA,EAEjB,YAAY,UAA6B,UAAoB,WAAqB,CAAC,GAAG,WAAoB;AACxG,SAAK,WAAW;AAChB,SAAK,WAAW,SAAS,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,EAAE,OAAO,OAAO;AACnE,SAAK,YAAY,aACZ,MAAM;AAAE,UAAI;AAAE,eAAO,IAAI,OAAO,WAAW,GAAG;AAAA,MAAG,QAAQ;AAAE,eAAO;AAAA,MAAM;AAAA,IAAE,GAAG,IAC9E;AACJ,eAAW,OAAO,SAAU,MAAK,KAAK,GAAG;AAAA,EAC3C;AAAA,EAEA,IAAI,SAAiB;AACnB,WAAO,KAAK,MAAM;AAAA,EACpB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,KAAK,KAAa,UAA8C;AAC9D,QAAI,KAAK,aAAa,CAAC,KAAK,UAAU,KAAK,GAAG,EAAG;AACjD,UAAM,cACJ,KAAK,aAAa,gBAClB,UAAU,mBAAmB,UAC7B,SAAS,iBAAiB,IACtB,IACA;AACN,SAAK,MAAM,KAAK,EAAE,KAAK,OAAO,KAAK,aAAa,eAAe,SAAS,KAAK,KAAK,QAAQ,IAAI,cAAc,EAAE,CAAC;AAAA,EACjH;AAAA;AAAA,EAGA,QAA4B;AAC1B,QAAI,KAAK,MAAM,WAAW,EAAG,QAAO;AACpC,YAAQ,KAAK,UAAU;AAAA,MACrB,KAAK;AACH,eAAO,KAAK,MAAM,IAAI,EAAG;AAAA,MAC3B,KAAK,cAAc;AACjB,YAAI,UAAU;AACd,iBAAS,IAAI,GAAG,IAAI,KAAK,MAAM,QAAQ,KAAK;AAC1C,cAAI,KAAK,MAAM,CAAC,EAAE,QAAQ,KAAK,MAAM,OAAO,EAAE,MAAO,WAAU;AAAA,QACjE;AACA,eAAO,KAAK,MAAM,OAAO,SAAS,CAAC,EAAE,CAAC,EAAE;AAAA,MAC1C;AAAA,MACA,KAAK;AAAA,MACL;AACE,eAAO,KAAK,MAAM,MAAM,EAAG;AAAA,IAC/B;AAAA,EACF;AAAA,EAEA,OAAO,WAA+C;AACpD,WAAO,KAAK,MAAM,OAAO,CAAC,MAAM,UAAU,EAAE,GAAG,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,GAAG;AAAA,EACpE;AAAA;AAAA,EAGA,gBAAgB,SAAyB;AACvC,QAAI;AACF,YAAM,KAAK,IAAI,OAAO,SAAS,GAAG;AAClC,YAAM,SAAS,KAAK,MAAM;AAC1B,WAAK,QAAQ,KAAK,MAAM,OAAO,CAAC,MAAM,GAAG,KAAK,EAAE,GAAG,CAAC;AACpD,aAAO,SAAS,KAAK,MAAM;AAAA,IAC7B,QAAQ;AACN,aAAO;AAAA,IACT;AAAA,EACF;AACF;",
+  "names": []
+}

package/dist/gap-pass.d.ts

@@ -0,0 +1,35 @@
+/**
+ * P6: Gap Pass — LLM-guided second crawl pass
+ *
+ * After the primary BFS crawl completes, this module:
+ * 1. Sends the discovered screen list to an LLM for gap analysis
+ * 2. HEAD-validates each suggested URL
+ * 3. Returns confirmed gap URLs for targeted re-crawl
+ *
+ * The caller (crawler.ts) is responsible for actually visiting the confirmed
+ * URLs with the existing browser — this module only identifies them.
+ */
+export interface GapCandidate {
+    category: string;
+    urlPatterns: string[];
+    confidence: number;
+    rationale: string;
+    confirmedUrl: string | null;
+    headStatus: number | null;
+}
+interface DiscoveredScreen {
+    url: string;
+    name?: string;
+    elements?: number;
+}
+interface LlmCfg {
+    provider: string;
+    model: string;
+    apiKey?: string;
+}
+/**
+ * Run LLM gap analysis + HEAD validation.
+ * Returns candidates with `confirmedUrl` set for those where HEAD returned 200.
+ */
+export declare function runGapAnalysis(screens: DiscoveredScreen[], appUrl: string, llmCfg: LlmCfg | null | undefined): Promise<GapCandidate[]>;
+export {};

package/dist/gap-pass.js

@@ -0,0 +1,81 @@
+import { logger as rootLogger } from "./logger.js";
+const logger = rootLogger.child({ module: "gap-pass" });
+async function runGapAnalysis(screens, appUrl, llmCfg) {
+  if (!llmCfg?.apiKey || screens.length < 2) return [];
+  const origin = (() => {
+    try {
+      return new URL(appUrl).origin;
+    } catch {
+      return "";
+    }
+  })();
+  if (!origin) return [];
+  const screenList = screens.slice(0, 80).map((s) => {
+    try {
+      return new URL(s.url).pathname + (s.name ? ` (${s.name})` : "");
+    } catch {
+      return s.url;
+    }
+  }).join("\n");
+  const prompt = `You are analyzing the screen map of a web application to find structural coverage gaps.
+
+Discovered screens (${screens.length} total):
+${screenList}
+
+App URL: ${appUrl}
+
+Based on these screens and common patterns for this type of web application, identify screen categories that are likely present but missing. Focus on pages that would exist in a typical SaaS or web app but are not represented above.
+
+Respond with ONLY valid JSON, no markdown:
+{"gaps":[{"category":"User profile edit","urlPatterns":["/profile/edit","/account/profile"],"confidence":0.8,"rationale":"Profile view found but no edit page"},{"category":"Billing history","urlPatterns":["/settings/billing","/account/billing"],"confidence":0.7,"rationale":"Settings section found but no billing page"}]}`;
+  try {
+    const Anthropic = (await import("@anthropic-ai/sdk")).default;
+    const client = new Anthropic({ apiKey: llmCfg.apiKey });
+    const response = await client.messages.create({
+      model: llmCfg.model ?? "claude-haiku-4-5-20251001",
+      max_tokens: 1024,
+      // @ts-ignore
+      system: [{ type: "text", text: "You are a web application coverage analyst. Respond with valid JSON only.", cache_control: { type: "ephemeral" } }],
+      messages: [{ role: "user", content: prompt }]
+    });
+    const text = response.content[0]?.text ?? "";
+    const jsonMatch = text.match(/\{[\s\S]*\}/);
+    if (!jsonMatch) return [];
+    const parsed = JSON.parse(jsonMatch[0]);
+    const rawGaps = parsed.gaps ?? [];
+    if (!Array.isArray(rawGaps) || rawGaps.length === 0) return [];
+    logger.info({ gaps: rawGaps.length }, "[P6] Gap analysis: LLM returned candidates");
+    const results = [];
+    for (const gap of rawGaps.slice(0, 15)) {
+      let confirmedUrl = null;
+      let headStatus = null;
+      for (const pattern of (gap.urlPatterns ?? []).slice(0, 4)) {
+        const fullUrl = pattern.startsWith("http") ? pattern : `${origin}${pattern.startsWith("/") ? "" : "/"}${pattern}`;
+        try {
+          const controller = new AbortController();
+          const timeout = setTimeout(() => controller.abort(), 6e3);
+          const res = await fetch(fullUrl, { method: "HEAD", signal: controller.signal, redirect: "follow" }).catch(() => null);
+          clearTimeout(timeout);
+          headStatus = res?.status ?? null;
+          if (res?.ok) {
+            confirmedUrl = fullUrl;
+            break;
+          }
+        } catch {
+        }
+      }
+      results.push({ category: gap.category, urlPatterns: gap.urlPatterns, confidence: gap.confidence, rationale: gap.rationale, confirmedUrl, headStatus });
+      logger.info({ category: gap.category, confirmedUrl, headStatus }, "[P6] Gap candidate checked");
+    }
+    const confirmed = results.filter((r) => r.confirmedUrl).length;
+    logger.info({ total: results.length, confirmed }, "[P6] Gap pass analysis complete");
+    return results;
+  } catch (err) {
+    logger.warn({ err: String(err) }, "[P6] Gap analysis failed \u2014 non-fatal");
+    return [];
+  }
+}
+export {
+  runGapAnalysis
+};
+//# sourceMappingURL=gap-pass.js.map

package/dist/gap-pass.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/gap-pass.ts"],
+  "sourcesContent": ["/**\n * P6: Gap Pass \u2014 LLM-guided second crawl pass\n *\n * After the primary BFS crawl completes, this module:\n * 1. Sends the discovered screen list to an LLM for gap analysis\n * 2. HEAD-validates each suggested URL\n * 3. Returns confirmed gap URLs for targeted re-crawl\n *\n * The caller (crawler.ts) is responsible for actually visiting the confirmed\n * URLs with the existing browser \u2014 this module only identifies them.\n */\n\nimport { logger as rootLogger } from './logger.js';\n\nconst logger = rootLogger.child({ module: 'gap-pass' });\n\nexport interface GapCandidate {\n  category: string;\n  urlPatterns: string[];\n  confidence: number;\n  rationale: string;\n  confirmedUrl: string | null;\n  headStatus: number | null;\n}\n\ninterface DiscoveredScreen {\n  url: string;\n  name?: string;\n  elements?: number;\n}\n\ninterface LlmCfg {\n  provider: string;\n  model: string;\n  apiKey?: string;\n}\n\n/**\n * Run LLM gap analysis + HEAD validation.\n * Returns candidates with `confirmedUrl` set for those where HEAD returned 200.\n */\nexport async function runGapAnalysis(\n  screens: DiscoveredScreen[],\n  appUrl: string,\n  llmCfg: LlmCfg | null | undefined,\n): Promise<GapCandidate[]> {\n  if (!llmCfg?.apiKey || screens.length < 2) return [];\n\n  const origin = (() => { try { return new URL(appUrl).origin; } catch { return ''; } })();\n  if (!origin) return [];\n\n  // Cap context \u2014 only send url + name, not full element data\n  const screenList = screens\n    .slice(0, 80)\n    .map(s => {\n      try { return new URL(s.url).pathname + (s.name ? ` (${s.name})` : ''); } catch { return s.url; }\n    })\n    .join('\\n');\n\n  const prompt = `You are analyzing the screen map of a web application to find structural coverage gaps.\n\nDiscovered screens (${screens.length} total):\n${screenList}\n\nApp URL: ${appUrl}\n\nBased on these screens and common patterns for this type of web application, identify screen categories that are likely present but missing. Focus on pages that would exist in a typical SaaS or web app but are not represented above.\n\nRespond with ONLY valid JSON, no markdown:\n{\"gaps\":[{\"category\":\"User profile edit\",\"urlPatterns\":[\"/profile/edit\",\"/account/profile\"],\"confidence\":0.8,\"rationale\":\"Profile view found but no edit page\"},{\"category\":\"Billing history\",\"urlPatterns\":[\"/settings/billing\",\"/account/billing\"],\"confidence\":0.7,\"rationale\":\"Settings section found but no billing page\"}]}`;\n\n  try {\n    const Anthropic = (await import('@anthropic-ai/sdk')).default;\n    const client = new Anthropic({ apiKey: llmCfg.apiKey });\n    const response = await client.messages.create({\n      model: llmCfg.model ?? 'claude-haiku-4-5-20251001',\n      max_tokens: 1024,\n      // @ts-ignore\n      system: [{ type: 'text', text: 'You are a web application coverage analyst. Respond with valid JSON only.', cache_control: { type: 'ephemeral' } }],\n      messages: [{ role: 'user', content: prompt }],\n    });\n\n    const text = (response.content[0] as any)?.text ?? '';\n    const jsonMatch = text.match(/\\{[\\s\\S]*\\}/);\n    if (!jsonMatch) return [];\n    const parsed = JSON.parse(jsonMatch[0]);\n    const rawGaps: Array<{ category: string; urlPatterns: string[]; confidence: number; rationale: string }> =\n      parsed.gaps ?? [];\n\n    if (!Array.isArray(rawGaps) || rawGaps.length === 0) return [];\n    logger.info({ gaps: rawGaps.length }, '[P6] Gap analysis: LLM returned candidates');\n\n    // HEAD-validate each URL pattern, take the first 200\n    const results: GapCandidate[] = [];\n    for (const gap of rawGaps.slice(0, 15)) {\n      let confirmedUrl: string | null = null;\n      let headStatus: number | null = null;\n\n      for (const pattern of (gap.urlPatterns ?? []).slice(0, 4)) {\n        const fullUrl = pattern.startsWith('http') ? pattern : `${origin}${pattern.startsWith('/') ? '' : '/'}${pattern}`;\n        try {\n          const controller = new AbortController();\n          const timeout = setTimeout(() => controller.abort(), 6_000);\n          const res = await fetch(fullUrl, { method: 'HEAD', signal: controller.signal, redirect: 'follow' }).catch(() => null);\n          clearTimeout(timeout);\n          headStatus = res?.status ?? null;\n          if (res?.ok) { confirmedUrl = fullUrl; break; }\n        } catch { /* non-fatal */ }\n      }\n\n      results.push({ category: gap.category, urlPatterns: gap.urlPatterns, confidence: gap.confidence, rationale: gap.rationale, confirmedUrl, headStatus });\n      logger.info({ category: gap.category, confirmedUrl, headStatus }, '[P6] Gap candidate checked');\n    }\n\n    const confirmed = results.filter(r => r.confirmedUrl).length;\n    logger.info({ total: results.length, confirmed }, '[P6] Gap pass analysis complete');\n    return results;\n  } catch (err) {\n    logger.warn({ err: String(err) }, '[P6] Gap analysis failed \u2014 non-fatal');\n    return [];\n  }\n}\n"],
+  "mappings": "AAYA,SAAS,UAAU,kBAAkB;AAErC,MAAM,SAAS,WAAW,MAAM,EAAE,QAAQ,WAAW,CAAC;AA2BtD,eAAsB,eACpB,SACA,QACA,QACyB;AACzB,MAAI,CAAC,QAAQ,UAAU,QAAQ,SAAS,EAAG,QAAO,CAAC;AAEnD,QAAM,UAAU,MAAM;AAAE,QAAI;AAAE,aAAO,IAAI,IAAI,MAAM,EAAE;AAAA,IAAQ,QAAQ;AAAE,aAAO;AAAA,IAAI;AAAA,EAAE,GAAG;AACvF,MAAI,CAAC,OAAQ,QAAO,CAAC;AAGrB,QAAM,aAAa,QAChB,MAAM,GAAG,EAAE,EACX,IAAI,OAAK;AACR,QAAI;AAAE,aAAO,IAAI,IAAI,EAAE,GAAG,EAAE,YAAY,EAAE,OAAO,KAAK,EAAE,IAAI,MAAM;AAAA,IAAK,QAAQ;AAAE,aAAO,EAAE;AAAA,IAAK;AAAA,EACjG,CAAC,EACA,KAAK,IAAI;AAEZ,QAAM,SAAS;AAAA;AAAA,sBAEK,QAAQ,MAAM;AAAA,EAClC,UAAU;AAAA;AAAA,WAED,MAAM;AAAA;AAAA;AAAA;AAAA;AAAA;AAOf,MAAI;AACF,UAAM,aAAa,MAAM,OAAO,mBAAmB,GAAG;AACtD,UAAM,SAAS,IAAI,UAAU,EAAE,QAAQ,OAAO,OAAO,CAAC;AACtD,UAAM,WAAW,MAAM,OAAO,SAAS,OAAO;AAAA,MAC5C,OAAO,OAAO,SAAS;AAAA,MACvB,YAAY;AAAA;AAAA,MAEZ,QAAQ,CAAC,EAAE,MAAM,QAAQ,MAAM,6EAA6E,eAAe,EAAE,MAAM,YAAY,EAAE,CAAC;AAAA,MAClJ,UAAU,CAAC,EAAE,MAAM,QAAQ,SAAS,OAAO,CAAC;AAAA,IAC9C,CAAC;AAED,UAAM,OAAQ,SAAS,QAAQ,CAAC,GAAW,QAAQ;AACnD,UAAM,YAAY,KAAK,MAAM,aAAa;AAC1C,QAAI,CAAC,UAAW,QAAO,CAAC;AACxB,UAAM,SAAS,KAAK,MAAM,UAAU,CAAC,CAAC;AACtC,UAAM,UACJ,OAAO,QAAQ,CAAC;AAElB,QAAI,CAAC,MAAM,QAAQ,OAAO,KAAK,QAAQ,WAAW,EAAG,QAAO,CAAC;AAC7D,WAAO,KAAK,EAAE,MAAM,QAAQ,OAAO,GAAG,4CAA4C;AAGlF,UAAM,UAA0B,CAAC;AACjC,eAAW,OAAO,QAAQ,MAAM,GAAG,EAAE,GAAG;AACtC,UAAI,eAA8B;AAClC,UAAI,aAA4B;AAEhC,iBAAW,YAAY,IAAI,eAAe,CAAC,GAAG,MAAM,GAAG,CAAC,GAAG;AACzD,cAAM,UAAU,QAAQ,WAAW,MAAM,IAAI,UAAU,GAAG,MAAM,GAAG,QAAQ,WAAW,GAAG,IAAI,KAAK,GAAG,GAAG,OAAO;AAC/G,YAAI;AACF,gBAAM,aAAa,IAAI,gBAAgB;AACvC,gBAAM,UAAU,WAAW,MAAM,WAAW,MAAM,GAAG,GAAK;AAC1D,gBAAM,MAAM,MAAM,MAAM,SAAS,EAAE,QAAQ,QAAQ,QAAQ,WAAW,QAAQ,UAAU,SAAS,CAAC,EAAE,MAAM,MAAM,IAAI;AACpH,uBAAa,OAAO;AACpB,uBAAa,KAAK,UAAU;AAC5B,cAAI,KAAK,IAAI;AAAE,2BAAe;AAAS;AAAA,UAAO;AAAA,QAChD,QAAQ;AAAA,QAAkB;AAAA,MAC5B;AAEA,cAAQ,KAAK,EAAE,UAAU,IAAI,UAAU,aAAa,IAAI,aAAa,YAAY,IAAI,YAAY,WAAW,IAAI,WAAW,cAAc,WAAW,CAAC;AACrJ,aAAO,KAAK,EAAE,UAAU,IAAI,UAAU,cAAc,WAAW,GAAG,4BAA4B;AAAA,IAChG;AAEA,UAAM,YAAY,QAAQ,OAAO,OAAK,EAAE,YAAY,EAAE;AACtD,WAAO,KAAK,EAAE,OAAO,QAAQ,QAAQ,UAAU,GAAG,iCAAiC;AACnF,WAAO;AAAA,EACT,SAAS,KAAK;AACZ,WAAO,KAAK,EAAE,KAAK,OAAO,GAAG,EAAE,GAAG,2CAAsC;AACxE,WAAO,CAAC;AAAA,EACV;AACF;",
+  "names": []
+}

package/dist/graphql-detector.d.ts

@@ -0,0 +1,39 @@
+/**
+ * GraphQL detection + introspection.
+ *
+ * During crawl, if a GraphQL endpoint is discovered (via network observer or
+ * known-URL heuristics), fire a minimal introspection query to get the schema.
+ * The schema is stored in the crawl artifact for use during test generation.
+ */
+export interface GqlType {
+    name: string;
+    kind: string;
+    fields?: Array<{
+        name: string;
+        type: string;
+    }>;
+    inputFields?: Array<{
+        name: string;
+        type: string;
+    }>;
+    enumValues?: string[];
+}
+export interface GqlSchema {
+    queryType?: string;
+    mutationType?: string;
+    subscriptionType?: string;
+    types: GqlType[];
+}
+export interface GqlIntrospectionResult {
+    endpoint: string;
+    schema: GqlSchema | null;
+    error?: string;
+    durationMs: number;
+}
+export declare function introspectGraphQL(endpoint: string, opts?: {
+    headers?: Record<string, string>;
+    timeoutMs?: number;
+    cookies?: string;
+}): Promise<GqlIntrospectionResult>;
+/** Guess GraphQL endpoint URLs from a known app URL. */
+export declare function guessGraphQLEndpoints(appUrl: string): string[];

package/dist/graphql-detector.js

@@ -0,0 +1,86 @@
+const INTROSPECTION_QUERY = `
+  query IntrospectionQuery {
+    __schema {
+      queryType { name }
+      mutationType { name }
+      subscriptionType { name }
+      types {
+        name
+        kind
+        fields(includeDeprecated: false) { name type { name kind ofType { name kind } } }
+        inputFields { name type { name kind ofType { name kind } } }
+        enumValues(includeDeprecated: false) { name }
+      }
+    }
+  }
+`;
+function flattenType(t) {
+  if (!t) return "unknown";
+  if (t.kind === "NON_NULL" || t.kind === "LIST") return flattenType(t.ofType);
+  return t.name ?? "unknown";
+}
+function parseSchema(raw) {
+  const s = raw?.__schema ?? raw?.data?.__schema ?? raw;
+  const types = (s?.types ?? []).filter((t) => !t.name?.startsWith("__")).map((t) => ({
+    name: t.name,
+    kind: t.kind,
+    fields: (t.fields ?? []).map((f) => ({ name: f.name, type: flattenType(f.type) })),
+    inputFields: (t.inputFields ?? []).map((f) => ({ name: f.name, type: flattenType(f.type) })),
+    enumValues: (t.enumValues ?? []).map((e) => e.name)
+  }));
+  return {
+    queryType: s?.queryType?.name,
+    mutationType: s?.mutationType?.name,
+    subscriptionType: s?.subscriptionType?.name,
+    types
+  };
+}
+async function introspectGraphQL(endpoint, opts = {}) {
+  const { headers = {}, timeoutMs = 1e4, cookies } = opts;
+  const t0 = Date.now();
+  try {
+    const reqHeaders = {
+      "Content-Type": "application/json",
+      "Accept": "application/json",
+      ...headers
+    };
+    if (cookies) reqHeaders["Cookie"] = cookies;
+    const res = await fetch(endpoint, {
+      method: "POST",
+      headers: reqHeaders,
+      body: JSON.stringify({ query: INTROSPECTION_QUERY }),
+      signal: AbortSignal.timeout(timeoutMs)
+    });
+    if (!res.ok) {
+      return { endpoint, schema: null, error: `HTTP ${res.status}`, durationMs: Date.now() - t0 };
+    }
+    const json = await res.json();
+    if (json?.errors?.length) {
+      const msg = json.errors[0]?.message ?? "introspection disabled";
+      return { endpoint, schema: null, error: msg, durationMs: Date.now() - t0 };
+    }
+    return { endpoint, schema: parseSchema(json?.data ?? json), durationMs: Date.now() - t0 };
+  } catch (e) {
+    return { endpoint, schema: null, error: e.message, durationMs: Date.now() - t0 };
+  }
+}
+function guessGraphQLEndpoints(appUrl) {
+  try {
+    const u = new URL(appUrl);
+    const base = `${u.protocol}//${u.host}`;
+    return [
+      `${base}/graphql`,
+      `${base}/api/graphql`,
+      `${base}/v1/graphql`,
+      `${base}/gql`,
+      `${base}/query`
+    ];
+  } catch {
+    return [];
+  }
+}
+export {
+  guessGraphQLEndpoints,
+  introspectGraphQL
+};
+//# sourceMappingURL=graphql-detector.js.map

package/dist/graphql-detector.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/graphql-detector.ts"],
+  "sourcesContent": ["/**\n * GraphQL detection + introspection.\n *\n * During crawl, if a GraphQL endpoint is discovered (via network observer or\n * known-URL heuristics), fire a minimal introspection query to get the schema.\n * The schema is stored in the crawl artifact for use during test generation.\n */\n\nexport interface GqlType {\n  name: string;\n  kind: string;\n  fields?: Array<{ name: string; type: string }>;\n  inputFields?: Array<{ name: string; type: string }>;\n  enumValues?: string[];\n}\n\nexport interface GqlSchema {\n  queryType?: string;\n  mutationType?: string;\n  subscriptionType?: string;\n  types: GqlType[];\n}\n\nexport interface GqlIntrospectionResult {\n  endpoint: string;\n  schema: GqlSchema | null;\n  error?: string;\n  durationMs: number;\n}\n\nconst INTROSPECTION_QUERY = `\n  query IntrospectionQuery {\n    __schema {\n      queryType { name }\n      mutationType { name }\n      subscriptionType { name }\n      types {\n        name\n        kind\n        fields(includeDeprecated: false) { name type { name kind ofType { name kind } } }\n        inputFields { name type { name kind ofType { name kind } } }\n        enumValues(includeDeprecated: false) { name }\n      }\n    }\n  }\n`;\n\nfunction flattenType(t: any): string {\n  if (!t) return 'unknown';\n  if (t.kind === 'NON_NULL' || t.kind === 'LIST') return flattenType(t.ofType);\n  return t.name ?? 'unknown';\n}\n\nfunction parseSchema(raw: any): GqlSchema {\n  const s = raw?.__schema ?? raw?.data?.__schema ?? raw;\n  const types: GqlType[] = (s?.types ?? [])\n    .filter((t: any) => !t.name?.startsWith('__'))\n    .map((t: any) => ({\n      name: t.name,\n      kind: t.kind,\n      fields: (t.fields ?? []).map((f: any) => ({ name: f.name, type: flattenType(f.type) })),\n      inputFields: (t.inputFields ?? []).map((f: any) => ({ name: f.name, type: flattenType(f.type) })),\n      enumValues: (t.enumValues ?? []).map((e: any) => e.name),\n    }));\n  return {\n    queryType: s?.queryType?.name,\n    mutationType: s?.mutationType?.name,\n    subscriptionType: s?.subscriptionType?.name,\n    types,\n  };\n}\n\nexport async function introspectGraphQL(\n  endpoint: string,\n  opts: {\n    headers?: Record<string, string>;\n    timeoutMs?: number;\n    cookies?: string;\n  } = {},\n): Promise<GqlIntrospectionResult> {\n  const { headers = {}, timeoutMs = 10_000, cookies } = opts;\n  const t0 = Date.now();\n  try {\n    const reqHeaders: Record<string, string> = {\n      'Content-Type': 'application/json',\n      'Accept': 'application/json',\n      ...headers,\n    };\n    if (cookies) reqHeaders['Cookie'] = cookies;\n\n    const res = await fetch(endpoint, {\n      method: 'POST',\n      headers: reqHeaders,\n      body: JSON.stringify({ query: INTROSPECTION_QUERY }),\n      signal: AbortSignal.timeout(timeoutMs),\n    });\n\n    if (!res.ok) {\n      return { endpoint, schema: null, error: `HTTP ${res.status}`, durationMs: Date.now() - t0 };\n    }\n\n    const json = await res.json();\n    if (json?.errors?.length) {\n      const msg = json.errors[0]?.message ?? 'introspection disabled';\n      return { endpoint, schema: null, error: msg, durationMs: Date.now() - t0 };\n    }\n\n    return { endpoint, schema: parseSchema(json?.data ?? json), durationMs: Date.now() - t0 };\n  } catch (e: any) {\n    return { endpoint, schema: null, error: e.message, durationMs: Date.now() - t0 };\n  }\n}\n\n/** Guess GraphQL endpoint URLs from a known app URL. */\nexport function guessGraphQLEndpoints(appUrl: string): string[] {\n  try {\n    const u = new URL(appUrl);\n    const base = `${u.protocol}//${u.host}`;\n    return [\n      `${base}/graphql`,\n      `${base}/api/graphql`,\n      `${base}/v1/graphql`,\n      `${base}/gql`,\n      `${base}/query`,\n    ];\n  } catch {\n    return [];\n  }\n}\n"],
+  "mappings": "AA8BA,MAAM,sBAAsB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAiB5B,SAAS,YAAY,GAAgB;AACnC,MAAI,CAAC,EAAG,QAAO;AACf,MAAI,EAAE,SAAS,cAAc,EAAE,SAAS,OAAQ,QAAO,YAAY,EAAE,MAAM;AAC3E,SAAO,EAAE,QAAQ;AACnB;AAEA,SAAS,YAAY,KAAqB;AACxC,QAAM,IAAI,KAAK,YAAY,KAAK,MAAM,YAAY;AAClD,QAAM,SAAoB,GAAG,SAAS,CAAC,GACpC,OAAO,CAAC,MAAW,CAAC,EAAE,MAAM,WAAW,IAAI,CAAC,EAC5C,IAAI,CAAC,OAAY;AAAA,IAChB,MAAM,EAAE;AAAA,IACR,MAAM,EAAE;AAAA,IACR,SAAS,EAAE,UAAU,CAAC,GAAG,IAAI,CAAC,OAAY,EAAE,MAAM,EAAE,MAAM,MAAM,YAAY,EAAE,IAAI,EAAE,EAAE;AAAA,IACtF,cAAc,EAAE,eAAe,CAAC,GAAG,IAAI,CAAC,OAAY,EAAE,MAAM,EAAE,MAAM,MAAM,YAAY,EAAE,IAAI,EAAE,EAAE;AAAA,IAChG,aAAa,EAAE,cAAc,CAAC,GAAG,IAAI,CAAC,MAAW,EAAE,IAAI;AAAA,EACzD,EAAE;AACJ,SAAO;AAAA,IACL,WAAW,GAAG,WAAW;AAAA,IACzB,cAAc,GAAG,cAAc;AAAA,IAC/B,kBAAkB,GAAG,kBAAkB;AAAA,IACvC;AAAA,EACF;AACF;AAEA,eAAsB,kBACpB,UACA,OAII,CAAC,GAC4B;AACjC,QAAM,EAAE,UAAU,CAAC,GAAG,YAAY,KAAQ,QAAQ,IAAI;AACtD,QAAM,KAAK,KAAK,IAAI;AACpB,MAAI;AACF,UAAM,aAAqC;AAAA,MACzC,gBAAgB;AAAA,MAChB,UAAU;AAAA,MACV,GAAG;AAAA,IACL;AACA,QAAI,QAAS,YAAW,QAAQ,IAAI;AAEpC,UAAM,MAAM,MAAM,MAAM,UAAU;AAAA,MAChC,QAAQ;AAAA,MACR,SAAS;AAAA,MACT,MAAM,KAAK,UAAU,EAAE,OAAO,oBAAoB,CAAC;AAAA,MACnD,QAAQ,YAAY,QAAQ,SAAS;AAAA,IACvC,CAAC;AAED,QAAI,CAAC,IAAI,IAAI;AACX,aAAO,EAAE,UAAU,QAAQ,MAAM,OAAO,QAAQ,IAAI,MAAM,IAAI,YAAY,KAAK,IAAI,IAAI,GAAG;AAAA,IAC5F;AAEA,UAAM,OAAO,MAAM,IAAI,KAAK;AAC5B,QAAI,MAAM,QAAQ,QAAQ;AACxB,YAAM,MAAM,KAAK,OAAO,CAAC,GAAG,WAAW;AACvC,aAAO,EAAE,UAAU,QAAQ,MAAM,OAAO,KAAK,YAAY,KAAK,IAAI,IAAI,GAAG;AAAA,IAC3E;AAEA,WAAO,EAAE,UAAU,QAAQ,YAAY,MAAM,QAAQ,IAAI,GAAG,YAAY,KAAK,IAAI,IAAI,GAAG;AAAA,EAC1F,SAAS,GAAQ;AACf,WAAO,EAAE,UAAU,QAAQ,MAAM,OAAO,EAAE,SAAS,YAAY,KAAK,IAAI,IAAI,GAAG;AAAA,EACjF;AACF;AAGO,SAAS,sBAAsB,QAA0B;AAC9D,MAAI;AACF,UAAM,IAAI,IAAI,IAAI,MAAM;AACxB,UAAM,OAAO,GAAG,EAAE,QAAQ,KAAK,EAAE,IAAI;AACrC,WAAO;AAAA,MACL,GAAG,IAAI;AAAA,MACP,GAAG,IAAI;AAAA,MACP,GAAG,IAAI;AAAA,MACP,GAAG,IAAI;AAAA,MACP,GAAG,IAAI;AAAA,IACT;AAAA,EACF,QAAQ;AACN,WAAO,CAAC;AAAA,EACV;AACF;",
+  "names": []
+}

package/dist/http-engine.d.ts

@@ -0,0 +1,80 @@
+/**
+ * HTTP-only crawl engine — 10-50x faster than browser for SSR/static sites.
+ *
+ * Uses native Node.js fetch (HTTP/2), linkedom, Readability, and Turndown.
+ * No browser overhead. Falls back to browser engine on JS-heavy sites.
+ *
+ * Features:
+ *   - HTTP/2 native (Node 20 built-in fetch)
+ *   - ETag / Last-Modified response caching
+ *   - 429 / 503 exponential backoff with jitter
+ *   - BFS / DFS traversal
+ *   - URL regex filtering
+ *   - Robots.txt compliance
+ *   - Link discovery (href, action, src)
+ *   - Markdown extraction (Readability + Turndown)
+ *   - OG / schema.org / meta extraction
+ *   - Table → JSON extraction
+ *   - JSON Lines, CSV, raw HTML output formats
+ *   - Streaming progress via onProgress callback
+ */
+export type HttpOutputFormat = 'json' | 'jsonlines' | 'csv' | 'markdown' | 'html';
+export interface HttpScreen {
+    url: string;
+    title: string | null;
+    markdown: string;
+    html?: string;
+    metadata: {
+        description: string | null;
+        ogTitle: string | null;
+        ogDescription: string | null;
+        ogImage: string | null;
+        canonical: string | null;
+        schemaTypes: string[];
+        lang: string | null;
+    };
+    tables: Array<Record<string, string>[]>;
+    links: string[];
+    images: Array<{
+        src: string;
+        alt: string;
+    }>;
+    statusCode: number;
+    fromCache: boolean;
+    requiresBrowser?: boolean;
+    durationMs: number;
+}
+export interface HttpCrawlOpts {
+    appUrl: string;
+    maxScreens?: number;
+    urlFilter?: string;
+    traversalStrategy?: 'BFS' | 'DFS';
+    outputFormats?: HttpOutputFormat[];
+    cacheDir?: string;
+    userAgent?: string;
+    headers?: Record<string, string>;
+    respectRobotsTxt?: boolean;
+    includeRawHtml?: boolean;
+    timeoutMs?: number;
+    concurrency?: number;
+    onProgress?: (update: {
+        url: string;
+        screensFound: number;
+        total: number;
+        cached: boolean;
+    }) => void;
+}
+export interface HttpCrawlResult {
+    screens: HttpScreen[];
+    stats: {
+        total: number;
+        cached: number;
+        errors: number;
+        rateLimited: number;
+        durationMs: number;
+        cacheHitRate: number;
+    };
+    jsonLines?: string;
+    csv?: string;
+}
+export declare function httpCrawl(opts: HttpCrawlOpts): Promise<HttpCrawlResult>;