@detiq/crawler 1.1.8 → 1.1.9

package/dist/email-otp.js

@@ -1,177 +1,218 @@
-const POLL_INTERVAL_MS = 2e3;
-const MAX_WAIT_MS = 6e4;
+/**
+ * Email OTP retrieval — supports 3 providers:
+ *   mailslurp  — managed SaaS (existing, no self-hosting)
+ *   imap       — any real IMAP mailbox (Gmail, Outlook, custom domain catch-all)
+ *   mailpit    — self-hosted SMTP trap (dev/staging only)
+ *
+ * Use mailslurp or imap for production apps.
+ * Use mailpit only when you can point the app's SMTP at the trap server.
+ */
+const POLL_INTERVAL_MS = 2_000;
+const MAX_WAIT_MS = 60_000;
+/** Extract OTP code from email body text. Prefers 6-digit, falls back to 4-8 digit. */
 function extractOTP(text) {
-  const m6 = text.match(/\b(\d{6})\b/);
-  if (m6) return m6[1];
-  const m48 = text.match(/\b(\d{4,8})\b/);
-  if (m48) return m48[1];
-  const magic = text.match(/\b([A-Z0-9]{6,10})\b/);
-  if (magic) return magic[1];
-  return null;
+    const m6 = text.match(/\b(\d{6})\b/);
+    if (m6)
+        return m6[1];
+    const m48 = text.match(/\b(\d{4,8})\b/);
+    if (m48)
+        return m48[1];
+    const magic = text.match(/\b([A-Z0-9]{6,10})\b/);
+    if (magic)
+        return magic[1];
+    return null;
 }
+/** Extract a magic-link URL from email body text. Matches /verify, /magic, /auth/callback, ?token= etc. */
 function extractMagicLink(text) {
-  const magicPath = /\/(verify|magic|login|auth|confirm|sign-in)[/?#]/i;
-  const magicQuery = /[?&](token|magic_token|login_token|auth_token|key)=/i;
-  const urlMatches = text.match(/https:\/\/[^\s<>"')\]]+/g);
-  if (!urlMatches) return null;
-  for (const raw of urlMatches) {
-    try {
-      const u = new URL(raw.replace(/[>'")\].,]+$/, ""));
-      if (magicPath.test(u.pathname) || magicQuery.test(u.search)) return u.toString();
-    } catch {
+    const magicPath = /\/(verify|magic|login|auth|confirm|sign-in)[/?#]/i;
+    const magicQuery = /[?&](token|magic_token|login_token|auth_token|key)=/i;
+    const urlMatches = text.match(/https:\/\/[^\s<>"')\]]+/g);
+    if (!urlMatches)
+        return null;
+    for (const raw of urlMatches) {
+        try {
+            const u = new URL(raw.replace(/[>'")\].,]+$/, ''));
+            if (magicPath.test(u.pathname) || magicQuery.test(u.search))
+                return u.toString();
+        }
+        catch { /* skip malformed URLs */ }
     }
-  }
-  return null;
+    return null;
 }
 function parseEmailContent(text) {
-  const magicLink = extractMagicLink(text);
-  if (magicLink) return { magicLink };
-  const otp = extractOTP(text);
-  if (otp) return { otp };
-  return {};
+    const magicLink = extractMagicLink(text);
+    if (magicLink)
+        return { magicLink };
+    const otp = extractOTP(text);
+    if (otp)
+        return { otp };
+    return {};
 }
-const MAILSLURP_BASE = "https://api.mailslurp.com";
+// ── MailSlurp ────────────────────────────────────────────────────────────────
+const MAILSLURP_BASE = 'https://api.mailslurp.com';
 async function waitForContentMailSlurp(apiKey, inboxId, since) {
-  const deadline = Date.now() + MAX_WAIT_MS;
-  while (Date.now() < deadline) {
-    await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS));
-    try {
-      const listRes = await fetch(
-        `${MAILSLURP_BASE}/inboxes/${inboxId}/emails?sort=DESC&limit=5&since=${encodeURIComponent(since)}`,
-        { headers: { "x-api-key": apiKey } }
-      );
-      if (!listRes.ok) continue;
-      const emails = await listRes.json();
-      if (!emails.length) continue;
-      const bodyRes = await fetch(`${MAILSLURP_BASE}/emails/${emails[0].id}`, { headers: { "x-api-key": apiKey } });
-      if (!bodyRes.ok) continue;
-      const email = await bodyRes.json();
-      const content = parseEmailContent((email.body ?? "") + " " + (email.subject ?? ""));
-      if (content.otp || content.magicLink) return content;
-    } catch {
+    const deadline = Date.now() + MAX_WAIT_MS;
+    while (Date.now() < deadline) {
+        await new Promise(r => setTimeout(r, POLL_INTERVAL_MS));
+        try {
+            const listRes = await fetch(`${MAILSLURP_BASE}/inboxes/${inboxId}/emails?sort=DESC&limit=5&since=${encodeURIComponent(since)}`, { headers: { 'x-api-key': apiKey } });
+            if (!listRes.ok)
+                continue;
+            const emails = await listRes.json();
+            if (!emails.length)
+                continue;
+            const bodyRes = await fetch(`${MAILSLURP_BASE}/emails/${emails[0].id}`, { headers: { 'x-api-key': apiKey } });
+            if (!bodyRes.ok)
+                continue;
+            const email = await bodyRes.json();
+            const content = parseEmailContent((email.body ?? '') + ' ' + (email.subject ?? ''));
+            if (content.otp || content.magicLink)
+                return content;
+        }
+        catch { /* retry */ }
     }
-  }
-  return null;
+    return null;
 }
 async function getInboxEmailMailSlurp(apiKey, inboxId) {
-  try {
-    const res = await fetch(`${MAILSLURP_BASE}/inboxes/${inboxId}`, { headers: { "x-api-key": apiKey } });
-    if (!res.ok) return null;
-    const data = await res.json();
-    return data.emailAddress ?? null;
-  } catch {
-    return null;
-  }
+    try {
+        const res = await fetch(`${MAILSLURP_BASE}/inboxes/${inboxId}`, { headers: { 'x-api-key': apiKey } });
+        if (!res.ok)
+            return null;
+        const data = await res.json();
+        return data.emailAddress ?? null;
+    }
+    catch {
+        return null;
+    }
 }
+// ── Mailpit (SMTP trap REST API) ──────────────────────────────────────────────
 async function waitForContentMailpit(baseUrl, since) {
-  const deadline = Date.now() + MAX_WAIT_MS;
-  const sinceMs = since.getTime();
-  while (Date.now() < deadline) {
-    await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS));
-    try {
-      const res = await fetch(`${baseUrl}/api/v1/messages?limit=10`);
-      if (!res.ok) continue;
-      const data = await res.json();
-      const recent = (data.messages ?? []).filter((m) => new Date(m.Created).getTime() > sinceMs);
-      for (const msg of recent) {
-        const snippet = parseEmailContent(msg.Snippet + " " + msg.Subject);
-        if (snippet.otp || snippet.magicLink) return snippet;
-        const bodyRes = await fetch(`${baseUrl}/api/v1/message/${msg.ID}`);
-        if (!bodyRes.ok) continue;
-        const body = await bodyRes.json();
-        const full = parseEmailContent((body.Text?.Body ?? "") + " " + (body.Subject ?? ""));
-        if (full.otp || full.magicLink) return full;
-      }
-    } catch {
+    const deadline = Date.now() + MAX_WAIT_MS;
+    const sinceMs = since.getTime();
+    while (Date.now() < deadline) {
+        await new Promise(r => setTimeout(r, POLL_INTERVAL_MS));
+        try {
+            const res = await fetch(`${baseUrl}/api/v1/messages?limit=10`);
+            if (!res.ok)
+                continue;
+            const data = await res.json();
+            const recent = (data.messages ?? []).filter(m => new Date(m.Created).getTime() > sinceMs);
+            for (const msg of recent) {
+                const snippet = parseEmailContent(msg.Snippet + ' ' + msg.Subject);
+                if (snippet.otp || snippet.magicLink)
+                    return snippet;
+                // Fetch full body — snippet may truncate the magic link
+                const bodyRes = await fetch(`${baseUrl}/api/v1/message/${msg.ID}`);
+                if (!bodyRes.ok)
+                    continue;
+                const body = await bodyRes.json();
+                const full = parseEmailContent((body.Text?.Body ?? '') + ' ' + (body.Subject ?? ''));
+                if (full.otp || full.magicLink)
+                    return full;
+            }
+        }
+        catch { /* retry */ }
     }
-  }
-  return null;
+    return null;
 }
+// ── IMAP (any mailbox) ────────────────────────────────────────────────────────
 async function waitForContentImap(cfg, since) {
-  let ImapFlow;
-  try {
-    const mod = await import("imapflow");
-    ImapFlow = mod.ImapFlow;
-  } catch {
-    throw new Error("IMAP provider requires imapflow package. Add it to dependencies.");
-  }
-  const client = new ImapFlow({
-    host: cfg.host,
-    port: cfg.port,
-    secure: cfg.tls,
-    auth: { user: cfg.user, pass: cfg.pass },
-    logger: false
-  });
-  try {
-    await client.connect();
-    const lock = await client.getMailboxLock("INBOX");
+    let ImapFlow;
     try {
-      const deadline = Date.now() + MAX_WAIT_MS;
-      while (Date.now() < deadline) {
-        const uids = await client.search({ seen: false, since });
-        if (uids.length) {
-          for await (const msg of client.fetch(uids.slice(-5), { source: true })) {
-            const { simpleParser } = await import("mailparser");
-            const parsed = await simpleParser(msg.source);
-            const text = (parsed.text ?? "") + " " + (parsed.subject ?? "");
-            const content = parseEmailContent(text);
-            if (content.otp || content.magicLink) return content;
-          }
-        }
-        await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS));
-      }
-      return null;
-    } finally {
-      lock.release();
+        const mod = await import('imapflow');
+        ImapFlow = mod.ImapFlow;
     }
-  } finally {
-    await client.logout().catch(() => {
+    catch {
+        throw new Error('IMAP provider requires imapflow package. Add it to dependencies.');
+    }
+    const client = new ImapFlow({
+        host: cfg.host,
+        port: cfg.port,
+        secure: cfg.tls,
+        auth: { user: cfg.user, pass: cfg.pass },
+        logger: false,
     });
-  }
+    try {
+        await client.connect();
+        const lock = await client.getMailboxLock('INBOX');
+        try {
+            const deadline = Date.now() + MAX_WAIT_MS;
+            while (Date.now() < deadline) {
+                const uids = await client.search({ seen: false, since });
+                if (uids.length) {
+                    for await (const msg of client.fetch(uids.slice(-5), { source: true })) {
+                        const { simpleParser } = await import('mailparser');
+                        const parsed = await simpleParser(msg.source);
+                        const text = (parsed.text ?? '') + ' ' + (parsed.subject ?? '');
+                        const content = parseEmailContent(text);
+                        if (content.otp || content.magicLink)
+                            return content;
+                    }
+                }
+                await new Promise(r => setTimeout(r, POLL_INTERVAL_MS));
+            }
+            return null;
+        }
+        finally {
+            lock.release();
+        }
+    }
+    finally {
+        await client.logout().catch(() => { });
+    }
 }
-async function waitForEmailContent(config, since) {
-  const sinceStr = typeof since === "string" ? since : since.toISOString();
-  const sinceDate = typeof since === "string" ? new Date(since) : since;
-  switch (config.provider) {
-    case "mailslurp":
-      if (!config.apiKey || !config.inboxId) throw new Error("mailslurp requires apiKey + inboxId");
-      return waitForContentMailSlurp(config.apiKey, config.inboxId, sinceStr);
-    case "mailpit":
-      return waitForContentMailpit(config.mailpitBaseUrl ?? "http://mailpit:8025", sinceDate);
-    case "imap":
-      if (!config.imapHost || !config.imapUser || !config.imapPass) {
-        throw new Error("imap requires imapHost, imapUser, imapPass");
-      }
-      return waitForContentImap({
-        host: config.imapHost,
-        port: config.imapPort ?? 993,
-        user: config.imapUser,
-        pass: config.imapPass,
-        tls: config.imapTls !== false
-      }, sinceDate);
-    default:
-      return null;
-  }
+// ── Public API ────────────────────────────────────────────────────────────────
+/**
+ * Wait for an email and return its OTP code or magic link URL.
+ * Prefer this over waitForEmailOTP when the app may send passwordless magic links.
+ */
+export async function waitForEmailContent(config, since) {
+    const sinceStr = typeof since === 'string' ? since : since.toISOString();
+    const sinceDate = typeof since === 'string' ? new Date(since) : since;
+    switch (config.provider) {
+        case 'mailslurp':
+            if (!config.apiKey || !config.inboxId)
+                throw new Error('mailslurp requires apiKey + inboxId');
+            return waitForContentMailSlurp(config.apiKey, config.inboxId, sinceStr);
+        case 'mailpit':
+            return waitForContentMailpit(config.mailpitBaseUrl ?? 'http://mailpit:8025', sinceDate);
+        case 'imap':
+            if (!config.imapHost || !config.imapUser || !config.imapPass) {
+                throw new Error('imap requires imapHost, imapUser, imapPass');
+            }
+            return waitForContentImap({
+                host: config.imapHost,
+                port: config.imapPort ?? 993,
+                user: config.imapUser,
+                pass: config.imapPass,
+                tls: config.imapTls !== false,
+            }, sinceDate);
+        default:
+            return null;
+    }
 }
-async function waitForEmailOTP(config, since) {
-  const content = await waitForEmailContent(config, since);
-  return content?.otp ?? null;
+/**
+ * Wait for an OTP email and extract the code.
+ * @deprecated Use waitForEmailContent which also handles magic links.
+ */
+export async function waitForEmailOTP(config, since) {
+    const content = await waitForEmailContent(config, since);
+    return content?.otp ?? null;
 }
-async function getInboxEmail(configOrApiKey, inboxId) {
-  if (typeof configOrApiKey === "string") {
-    return getInboxEmailMailSlurp(configOrApiKey, inboxId ?? "");
-  }
-  if (configOrApiKey.provider === "mailslurp" && configOrApiKey.apiKey && configOrApiKey.inboxId) {
-    return getInboxEmailMailSlurp(configOrApiKey.apiKey, configOrApiKey.inboxId);
-  }
-  if (configOrApiKey.provider === "imap") {
-    return configOrApiKey.imapUser ?? null;
-  }
-  return null;
+/**
+ * Get the email address for a MailSlurp inbox (used to display in UI).
+ * Returns null for IMAP/Mailpit providers — use imapUser directly.
+ */
+export async function getInboxEmail(configOrApiKey, inboxId) {
+    // Legacy call: getInboxEmail(apiKey, inboxId)
+    if (typeof configOrApiKey === 'string') {
+        return getInboxEmailMailSlurp(configOrApiKey, inboxId ?? '');
+    }
+    if (configOrApiKey.provider === 'mailslurp' && configOrApiKey.apiKey && configOrApiKey.inboxId) {
+        return getInboxEmailMailSlurp(configOrApiKey.apiKey, configOrApiKey.inboxId);
+    }
+    if (configOrApiKey.provider === 'imap') {
+        return configOrApiKey.imapUser ?? null;
+    }
+    return null;
 }
-export {
-  getInboxEmail,
-  waitForEmailContent,
-  waitForEmailOTP
-};
-//# sourceMappingURL=email-otp.js.map

package/dist/engine-router.js

@@ -1,94 +1,105 @@
-class RenderingTypePredictor {
-  cache = /* @__PURE__ */ new Map();
-  /** Return cached decision if available, undefined if unknown. */
-  getCached(url) {
-    try {
-      return this.cache.get(new URL(url).hostname);
-    } catch {
-      return void 0;
-    }
-  }
-  /**
-   * Analyze an HTTP response body to predict if JS rendering is required.
-   * Returns 'browser' if JS-heavy signals detected, 'http' otherwise.
-   */
-  predict(url, htmlBody, statusCode) {
-    let decision = "http";
-    try {
-      if (statusCode >= 400) {
-        decision = "browser";
-      } else {
-        const bodyLen = htmlBody.length;
-        const textOnly = htmlBody.replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
-        const wordCount = textOnly.split(/\s+/).filter((w) => w.length > 2).length;
-        const hasSpaRoot = /<div[^>]*\s+id=["'](?:root|app|__nuxt|__next|app-root)["'][^>]*>\s*<\/div>/i.test(htmlBody) || /<app-root[^>]*>\s*<\/app-root>/i.test(htmlBody);
-        const hasNoscriptFallback = /<noscript[^>]*>[\s\S]{100,}/i.test(htmlBody);
-        const tinyBody = bodyLen < 3e3 && wordCount < 50;
-        if (hasSpaRoot && wordCount < 100 || tinyBody && !hasNoscriptFallback) {
-          decision = "browser";
+/**
+ * Per-domain rendering predictor — learns whether a domain requires a full browser
+ * or can be served by the fast HTTP engine. Implements adaptive crawlee-python pattern.
+ *
+ * Strategy:
+ * 1. On first encounter of a domain, classify the HTTP response body.
+ * 2. Store the decision in memory (Map). All future URLs on same domain skip re-testing.
+ * 3. "Requires browser" signals: React/Vue/Angular root with no content, tiny body (<500 chars meaningful text).
+ */
+export class RenderingTypePredictor {
+    cache = new Map();
+    /** Return cached decision if available, undefined if unknown. */
+    getCached(url) {
+        try {
+            return this.cache.get(new URL(url).hostname);
+        }
+        catch {
+            return undefined;
         }
-      }
-    } catch {
     }
-    try {
-      this.cache.set(new URL(url).hostname, decision);
-    } catch {
+    /**
+     * Analyze an HTTP response body to predict if JS rendering is required.
+     * Returns 'browser' if JS-heavy signals detected, 'http' otherwise.
+     */
+    predict(url, htmlBody, statusCode) {
+        let decision = 'http';
+        try {
+            if (statusCode >= 400) {
+                decision = 'browser'; // let browser handle error pages
+            }
+            else {
+                const bodyLen = htmlBody.length;
+                // Strip tags to get raw text
+                const textOnly = htmlBody.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim();
+                const wordCount = textOnly.split(/\s+/).filter(w => w.length > 2).length;
+                // SPA signals: very short text, presence of React/Vue/Angular root markers
+                const hasSpaRoot = /<div[^>]*\s+id=["'](?:root|app|__nuxt|__next|app-root)["'][^>]*>\s*<\/div>/i.test(htmlBody)
+                    || /<app-root[^>]*>\s*<\/app-root>/i.test(htmlBody);
+                const hasNoscriptFallback = /<noscript[^>]*>[\s\S]{100,}/i.test(htmlBody);
+                const tinyBody = bodyLen < 3000 && wordCount < 50;
+                if ((hasSpaRoot && wordCount < 100) || (tinyBody && !hasNoscriptFallback)) {
+                    decision = 'browser';
+                }
+            }
+        }
+        catch { /* default http */ }
+        try {
+            this.cache.set(new URL(url).hostname, decision);
+        }
+        catch { /* ignore */ }
+        return decision;
     }
-    return decision;
-  }
-  /** Force a domain to use a specific engine (e.g. after observing crawl errors). */
-  set(url, engine) {
-    try {
-      this.cache.set(new URL(url).hostname, engine);
-    } catch {
+    /** Force a domain to use a specific engine (e.g. after observing crawl errors). */
+    set(url, engine) {
+        try {
+            this.cache.set(new URL(url).hostname, engine);
+        }
+        catch { /* ignore */ }
     }
-  }
-  stats() {
-    const browserDomains = [];
-    const httpDomains = [];
-    for (const [domain, engine] of this.cache) {
-      if (engine === "browser") browserDomains.push(domain);
-      else httpDomains.push(domain);
+    stats() {
+        const browserDomains = [];
+        const httpDomains = [];
+        for (const [domain, engine] of this.cache) {
+            if (engine === 'browser')
+                browserDomains.push(domain);
+            else
+                httpDomains.push(domain);
+        }
+        return { cached: this.cache.size, browserDomains, httpDomains };
     }
-    return { cached: this.cache.size, browserDomains, httpDomains };
-  }
 }
-const renderingPredictor = new RenderingTypePredictor();
-function chooseCrawlEngines(input) {
-  const engines = [];
-  const reasons = [];
-  const appType = input.appType?.toLowerCase();
-  if (appType === "android" || appType === "ios" || appType === "windows" || appType === "macos") {
-    return { engines: ["native"], reasons: [`appType=${input.appType}`], confidence: 0.95 };
-  }
-  if (input.hasStorageState || input.hasAuthToken) {
-    engines.push("recorded-session");
-    reasons.push("session/token auth available");
-  }
-  engines.push(input.hasProxy ? "browser-stealth" : "browser");
-  reasons.push(input.hasProxy ? "proxy configured; prefer stealth browser path" : "default browser path");
-  if (input.hasCredentials || input.profile.industry !== "unknown") {
-    engines.push("stagehand");
-    reasons.push(input.hasCredentials ? "credentials available for AI login/navigation" : `domain profile=${input.profile.industry}`);
-  }
-  const hardUi = input.profile.industry === "legacy-enterprise" || /captcha|blocked|canvas|iframe|bot/i.test(input.previousFailureReason ?? "");
-  if (input.hasSkyvern && hardUi) {
-    engines.push("vision");
-    reasons.push("hard UI or previous block detected; vision fallback enabled");
-  }
-  if (engines.length === 1) {
-    engines.unshift("http");
-    reasons.push("HTTP/static extraction can be attempted before browser escalation");
-  }
-  return {
-    engines: Array.from(new Set(engines)),
-    reasons,
-    confidence: input.profile.confidence
-  };
+/** Singleton predictor — shared across crawl jobs in the same process. */
+export const renderingPredictor = new RenderingTypePredictor();
+export function chooseCrawlEngines(input) {
+    const engines = [];
+    const reasons = [];
+    const appType = input.appType?.toLowerCase();
+    if (appType === 'android' || appType === 'ios' || appType === 'windows' || appType === 'macos') {
+        return { engines: ['native'], reasons: [`appType=${input.appType}`], confidence: 0.95 };
+    }
+    if (input.hasStorageState || input.hasAuthToken) {
+        engines.push('recorded-session');
+        reasons.push('session/token auth available');
+    }
+    engines.push(input.hasProxy ? 'browser-stealth' : 'browser');
+    reasons.push(input.hasProxy ? 'proxy configured; prefer stealth browser path' : 'default browser path');
+    if (input.hasCredentials || input.profile.industry !== 'unknown') {
+        engines.push('stagehand');
+        reasons.push(input.hasCredentials ? 'credentials available for AI login/navigation' : `domain profile=${input.profile.industry}`);
+    }
+    const hardUi = input.profile.industry === 'legacy-enterprise' || /captcha|blocked|canvas|iframe|bot/i.test(input.previousFailureReason ?? '');
+    if (input.hasSkyvern && hardUi) {
+        engines.push('vision');
+        reasons.push('hard UI or previous block detected; vision fallback enabled');
+    }
+    if (engines.length === 1) {
+        engines.unshift('http');
+        reasons.push('HTTP/static extraction can be attempted before browser escalation');
+    }
+    return {
+        engines: Array.from(new Set(engines)),
+        reasons,
+        confidence: input.profile.confidence,
+    };
 }
-export {
-  RenderingTypePredictor,
-  chooseCrawlEngines,
-  renderingPredictor
-};
-//# sourceMappingURL=engine-router.js.map