@detiq/crawler 1.1.9 → 1.1.11

package/dist/email-otp.js

@@ -1,218 +1,177 @@
-/**
- * Email OTP retrieval — supports 3 providers:
- *   mailslurp  — managed SaaS (existing, no self-hosting)
- *   imap       — any real IMAP mailbox (Gmail, Outlook, custom domain catch-all)
- *   mailpit    — self-hosted SMTP trap (dev/staging only)
- *
- * Use mailslurp or imap for production apps.
- * Use mailpit only when you can point the app's SMTP at the trap server.
- */
-const POLL_INTERVAL_MS = 2_000;
-const MAX_WAIT_MS = 60_000;
-/** Extract OTP code from email body text. Prefers 6-digit, falls back to 4-8 digit. */
+const POLL_INTERVAL_MS = 2e3;
+const MAX_WAIT_MS = 6e4;
 function extractOTP(text) {
-    const m6 = text.match(/\b(\d{6})\b/);
-    if (m6)
-        return m6[1];
-    const m48 = text.match(/\b(\d{4,8})\b/);
-    if (m48)
-        return m48[1];
-    const magic = text.match(/\b([A-Z0-9]{6,10})\b/);
-    if (magic)
-        return magic[1];
-    return null;
+  const m6 = text.match(/\b(\d{6})\b/);
+  if (m6) return m6[1];
+  const m48 = text.match(/\b(\d{4,8})\b/);
+  if (m48) return m48[1];
+  const magic = text.match(/\b([A-Z0-9]{6,10})\b/);
+  if (magic) return magic[1];
+  return null;
 }
-/** Extract a magic-link URL from email body text. Matches /verify, /magic, /auth/callback, ?token= etc. */
 function extractMagicLink(text) {
-    const magicPath = /\/(verify|magic|login|auth|confirm|sign-in)[/?#]/i;
-    const magicQuery = /[?&](token|magic_token|login_token|auth_token|key)=/i;
-    const urlMatches = text.match(/https:\/\/[^\s<>"')\]]+/g);
-    if (!urlMatches)
-        return null;
-    for (const raw of urlMatches) {
-        try {
-            const u = new URL(raw.replace(/[>'")\].,]+$/, ''));
-            if (magicPath.test(u.pathname) || magicQuery.test(u.search))
-                return u.toString();
-        }
-        catch { /* skip malformed URLs */ }
+  const magicPath = /\/(verify|magic|login|auth|confirm|sign-in)[/?#]/i;
+  const magicQuery = /[?&](token|magic_token|login_token|auth_token|key)=/i;
+  const urlMatches = text.match(/https:\/\/[^\s<>"')\]]+/g);
+  if (!urlMatches) return null;
+  for (const raw of urlMatches) {
+    try {
+      const u = new URL(raw.replace(/[>'")\].,]+$/, ""));
+      if (magicPath.test(u.pathname) || magicQuery.test(u.search)) return u.toString();
+    } catch {
     }
-    return null;
+  }
+  return null;
 }
 function parseEmailContent(text) {
-    const magicLink = extractMagicLink(text);
-    if (magicLink)
-        return { magicLink };
-    const otp = extractOTP(text);
-    if (otp)
-        return { otp };
-    return {};
+  const magicLink = extractMagicLink(text);
+  if (magicLink) return { magicLink };
+  const otp = extractOTP(text);
+  if (otp) return { otp };
+  return {};
 }
-// ── MailSlurp ────────────────────────────────────────────────────────────────
-const MAILSLURP_BASE = 'https://api.mailslurp.com';
+const MAILSLURP_BASE = "https://api.mailslurp.com";
 async function waitForContentMailSlurp(apiKey, inboxId, since) {
-    const deadline = Date.now() + MAX_WAIT_MS;
-    while (Date.now() < deadline) {
-        await new Promise(r => setTimeout(r, POLL_INTERVAL_MS));
-        try {
-            const listRes = await fetch(`${MAILSLURP_BASE}/inboxes/${inboxId}/emails?sort=DESC&limit=5&since=${encodeURIComponent(since)}`, { headers: { 'x-api-key': apiKey } });
-            if (!listRes.ok)
-                continue;
-            const emails = await listRes.json();
-            if (!emails.length)
-                continue;
-            const bodyRes = await fetch(`${MAILSLURP_BASE}/emails/${emails[0].id}`, { headers: { 'x-api-key': apiKey } });
-            if (!bodyRes.ok)
-                continue;
-            const email = await bodyRes.json();
-            const content = parseEmailContent((email.body ?? '') + ' ' + (email.subject ?? ''));
-            if (content.otp || content.magicLink)
-                return content;
-        }
-        catch { /* retry */ }
+  const deadline = Date.now() + MAX_WAIT_MS;
+  while (Date.now() < deadline) {
+    await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS));
+    try {
+      const listRes = await fetch(
+        `${MAILSLURP_BASE}/inboxes/${inboxId}/emails?sort=DESC&limit=5&since=${encodeURIComponent(since)}`,
+        { headers: { "x-api-key": apiKey } }
+      );
+      if (!listRes.ok) continue;
+      const emails = await listRes.json();
+      if (!emails.length) continue;
+      const bodyRes = await fetch(`${MAILSLURP_BASE}/emails/${emails[0].id}`, { headers: { "x-api-key": apiKey } });
+      if (!bodyRes.ok) continue;
+      const email = await bodyRes.json();
+      const content = parseEmailContent((email.body ?? "") + " " + (email.subject ?? ""));
+      if (content.otp || content.magicLink) return content;
+    } catch {
     }
-    return null;
+  }
+  return null;
 }
 async function getInboxEmailMailSlurp(apiKey, inboxId) {
-    try {
-        const res = await fetch(`${MAILSLURP_BASE}/inboxes/${inboxId}`, { headers: { 'x-api-key': apiKey } });
-        if (!res.ok)
-            return null;
-        const data = await res.json();
-        return data.emailAddress ?? null;
-    }
-    catch {
-        return null;
-    }
+  try {
+    const res = await fetch(`${MAILSLURP_BASE}/inboxes/${inboxId}`, { headers: { "x-api-key": apiKey } });
+    if (!res.ok) return null;
+    const data = await res.json();
+    return data.emailAddress ?? null;
+  } catch {
+    return null;
+  }
 }
-// ── Mailpit (SMTP trap REST API) ──────────────────────────────────────────────
 async function waitForContentMailpit(baseUrl, since) {
-    const deadline = Date.now() + MAX_WAIT_MS;
-    const sinceMs = since.getTime();
-    while (Date.now() < deadline) {
-        await new Promise(r => setTimeout(r, POLL_INTERVAL_MS));
-        try {
-            const res = await fetch(`${baseUrl}/api/v1/messages?limit=10`);
-            if (!res.ok)
-                continue;
-            const data = await res.json();
-            const recent = (data.messages ?? []).filter(m => new Date(m.Created).getTime() > sinceMs);
-            for (const msg of recent) {
-                const snippet = parseEmailContent(msg.Snippet + ' ' + msg.Subject);
-                if (snippet.otp || snippet.magicLink)
-                    return snippet;
-                // Fetch full body — snippet may truncate the magic link
-                const bodyRes = await fetch(`${baseUrl}/api/v1/message/${msg.ID}`);
-                if (!bodyRes.ok)
-                    continue;
-                const body = await bodyRes.json();
-                const full = parseEmailContent((body.Text?.Body ?? '') + ' ' + (body.Subject ?? ''));
-                if (full.otp || full.magicLink)
-                    return full;
-            }
-        }
-        catch { /* retry */ }
+  const deadline = Date.now() + MAX_WAIT_MS;
+  const sinceMs = since.getTime();
+  while (Date.now() < deadline) {
+    await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS));
+    try {
+      const res = await fetch(`${baseUrl}/api/v1/messages?limit=10`);
+      if (!res.ok) continue;
+      const data = await res.json();
+      const recent = (data.messages ?? []).filter((m) => new Date(m.Created).getTime() > sinceMs);
+      for (const msg of recent) {
+        const snippet = parseEmailContent(msg.Snippet + " " + msg.Subject);
+        if (snippet.otp || snippet.magicLink) return snippet;
+        const bodyRes = await fetch(`${baseUrl}/api/v1/message/${msg.ID}`);
+        if (!bodyRes.ok) continue;
+        const body = await bodyRes.json();
+        const full = parseEmailContent((body.Text?.Body ?? "") + " " + (body.Subject ?? ""));
+        if (full.otp || full.magicLink) return full;
+      }
+    } catch {
     }
-    return null;
+  }
+  return null;
 }
-// ── IMAP (any mailbox) ────────────────────────────────────────────────────────
 async function waitForContentImap(cfg, since) {
-    let ImapFlow;
+  let ImapFlow;
+  try {
+    const mod = await import("imapflow");
+    ImapFlow = mod.ImapFlow;
+  } catch {
+    throw new Error("IMAP provider requires imapflow package. Add it to dependencies.");
+  }
+  const client = new ImapFlow({
+    host: cfg.host,
+    port: cfg.port,
+    secure: cfg.tls,
+    auth: { user: cfg.user, pass: cfg.pass },
+    logger: false
+  });
+  try {
+    await client.connect();
+    const lock = await client.getMailboxLock("INBOX");
     try {
-        const mod = await import('imapflow');
-        ImapFlow = mod.ImapFlow;
-    }
-    catch {
-        throw new Error('IMAP provider requires imapflow package. Add it to dependencies.');
-    }
-    const client = new ImapFlow({
-        host: cfg.host,
-        port: cfg.port,
-        secure: cfg.tls,
-        auth: { user: cfg.user, pass: cfg.pass },
-        logger: false,
-    });
-    try {
-        await client.connect();
-        const lock = await client.getMailboxLock('INBOX');
-        try {
-            const deadline = Date.now() + MAX_WAIT_MS;
-            while (Date.now() < deadline) {
-                const uids = await client.search({ seen: false, since });
-                if (uids.length) {
-                    for await (const msg of client.fetch(uids.slice(-5), { source: true })) {
-                        const { simpleParser } = await import('mailparser');
-                        const parsed = await simpleParser(msg.source);
-                        const text = (parsed.text ?? '') + ' ' + (parsed.subject ?? '');
-                        const content = parseEmailContent(text);
-                        if (content.otp || content.magicLink)
-                            return content;
-                    }
-                }
-                await new Promise(r => setTimeout(r, POLL_INTERVAL_MS));
-            }
-            return null;
-        }
-        finally {
-            lock.release();
+      const deadline = Date.now() + MAX_WAIT_MS;
+      while (Date.now() < deadline) {
+        const uids = await client.search({ seen: false, since });
+        if (uids.length) {
+          for await (const msg of client.fetch(uids.slice(-5), { source: true })) {
+            const { simpleParser } = await import("mailparser");
+            const parsed = await simpleParser(msg.source);
+            const text = (parsed.text ?? "") + " " + (parsed.subject ?? "");
+            const content = parseEmailContent(text);
+            if (content.otp || content.magicLink) return content;
+          }
         }
+        await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS));
+      }
+      return null;
+    } finally {
+      lock.release();
     }
-    finally {
-        await client.logout().catch(() => { });
-    }
+  } finally {
+    await client.logout().catch(() => {
+    });
+  }
 }
-// ── Public API ────────────────────────────────────────────────────────────────
-/**
- * Wait for an email and return its OTP code or magic link URL.
- * Prefer this over waitForEmailOTP when the app may send passwordless magic links.
- */
-export async function waitForEmailContent(config, since) {
-    const sinceStr = typeof since === 'string' ? since : since.toISOString();
-    const sinceDate = typeof since === 'string' ? new Date(since) : since;
-    switch (config.provider) {
-        case 'mailslurp':
-            if (!config.apiKey || !config.inboxId)
-                throw new Error('mailslurp requires apiKey + inboxId');
-            return waitForContentMailSlurp(config.apiKey, config.inboxId, sinceStr);
-        case 'mailpit':
-            return waitForContentMailpit(config.mailpitBaseUrl ?? 'http://mailpit:8025', sinceDate);
-        case 'imap':
-            if (!config.imapHost || !config.imapUser || !config.imapPass) {
-                throw new Error('imap requires imapHost, imapUser, imapPass');
-            }
-            return waitForContentImap({
-                host: config.imapHost,
-                port: config.imapPort ?? 993,
-                user: config.imapUser,
-                pass: config.imapPass,
-                tls: config.imapTls !== false,
-            }, sinceDate);
-        default:
-            return null;
-    }
+async function waitForEmailContent(config, since) {
+  const sinceStr = typeof since === "string" ? since : since.toISOString();
+  const sinceDate = typeof since === "string" ? new Date(since) : since;
+  switch (config.provider) {
+    case "mailslurp":
+      if (!config.apiKey || !config.inboxId) throw new Error("mailslurp requires apiKey + inboxId");
+      return waitForContentMailSlurp(config.apiKey, config.inboxId, sinceStr);
+    case "mailpit":
+      return waitForContentMailpit(config.mailpitBaseUrl ?? "http://mailpit:8025", sinceDate);
+    case "imap":
+      if (!config.imapHost || !config.imapUser || !config.imapPass) {
+        throw new Error("imap requires imapHost, imapUser, imapPass");
+      }
+      return waitForContentImap({
+        host: config.imapHost,
+        port: config.imapPort ?? 993,
+        user: config.imapUser,
+        pass: config.imapPass,
+        tls: config.imapTls !== false
+      }, sinceDate);
+    default:
+      return null;
+  }
 }
-/**
- * Wait for an OTP email and extract the code.
- * @deprecated Use waitForEmailContent which also handles magic links.
- */
-export async function waitForEmailOTP(config, since) {
-    const content = await waitForEmailContent(config, since);
-    return content?.otp ?? null;
+async function waitForEmailOTP(config, since) {
+  const content = await waitForEmailContent(config, since);
+  return content?.otp ?? null;
 }
-/**
- * Get the email address for a MailSlurp inbox (used to display in UI).
- * Returns null for IMAP/Mailpit providers — use imapUser directly.
- */
-export async function getInboxEmail(configOrApiKey, inboxId) {
-    // Legacy call: getInboxEmail(apiKey, inboxId)
-    if (typeof configOrApiKey === 'string') {
-        return getInboxEmailMailSlurp(configOrApiKey, inboxId ?? '');
-    }
-    if (configOrApiKey.provider === 'mailslurp' && configOrApiKey.apiKey && configOrApiKey.inboxId) {
-        return getInboxEmailMailSlurp(configOrApiKey.apiKey, configOrApiKey.inboxId);
-    }
-    if (configOrApiKey.provider === 'imap') {
-        return configOrApiKey.imapUser ?? null;
-    }
-    return null;
+async function getInboxEmail(configOrApiKey, inboxId) {
+  if (typeof configOrApiKey === "string") {
+    return getInboxEmailMailSlurp(configOrApiKey, inboxId ?? "");
+  }
+  if (configOrApiKey.provider === "mailslurp" && configOrApiKey.apiKey && configOrApiKey.inboxId) {
+    return getInboxEmailMailSlurp(configOrApiKey.apiKey, configOrApiKey.inboxId);
+  }
+  if (configOrApiKey.provider === "imap") {
+    return configOrApiKey.imapUser ?? null;
+  }
+  return null;
 }
+export {
+  getInboxEmail,
+  waitForEmailContent,
+  waitForEmailOTP
+};
+//# sourceMappingURL=email-otp.js.map

package/dist/engine-router.js

@@ -1,105 +1,94 @@
-/**
- * Per-domain rendering predictor — learns whether a domain requires a full browser
- * or can be served by the fast HTTP engine. Implements adaptive crawlee-python pattern.
- *
- * Strategy:
- * 1. On first encounter of a domain, classify the HTTP response body.
- * 2. Store the decision in memory (Map). All future URLs on same domain skip re-testing.
- * 3. "Requires browser" signals: React/Vue/Angular root with no content, tiny body (<500 chars meaningful text).
- */
-export class RenderingTypePredictor {
-    cache = new Map();
-    /** Return cached decision if available, undefined if unknown. */
-    getCached(url) {
-        try {
-            return this.cache.get(new URL(url).hostname);
-        }
-        catch {
-            return undefined;
-        }
+class RenderingTypePredictor {
+  cache = /* @__PURE__ */ new Map();
+  /** Return cached decision if available, undefined if unknown. */
+  getCached(url) {
+    try {
+      return this.cache.get(new URL(url).hostname);
+    } catch {
+      return void 0;
     }
-    /**
-     * Analyze an HTTP response body to predict if JS rendering is required.
-     * Returns 'browser' if JS-heavy signals detected, 'http' otherwise.
-     */
-    predict(url, htmlBody, statusCode) {
-        let decision = 'http';
-        try {
-            if (statusCode >= 400) {
-                decision = 'browser'; // let browser handle error pages
-            }
-            else {
-                const bodyLen = htmlBody.length;
-                // Strip tags to get raw text
-                const textOnly = htmlBody.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim();
-                const wordCount = textOnly.split(/\s+/).filter(w => w.length > 2).length;
-                // SPA signals: very short text, presence of React/Vue/Angular root markers
-                const hasSpaRoot = /<div[^>]*\s+id=["'](?:root|app|__nuxt|__next|app-root)["'][^>]*>\s*<\/div>/i.test(htmlBody)
-                    || /<app-root[^>]*>\s*<\/app-root>/i.test(htmlBody);
-                const hasNoscriptFallback = /<noscript[^>]*>[\s\S]{100,}/i.test(htmlBody);
-                const tinyBody = bodyLen < 3000 && wordCount < 50;
-                if ((hasSpaRoot && wordCount < 100) || (tinyBody && !hasNoscriptFallback)) {
-                    decision = 'browser';
-                }
-            }
-        }
-        catch { /* default http */ }
-        try {
-            this.cache.set(new URL(url).hostname, decision);
+  }
+  /**
+   * Analyze an HTTP response body to predict if JS rendering is required.
+   * Returns 'browser' if JS-heavy signals detected, 'http' otherwise.
+   */
+  predict(url, htmlBody, statusCode) {
+    let decision = "http";
+    try {
+      if (statusCode >= 400) {
+        decision = "browser";
+      } else {
+        const bodyLen = htmlBody.length;
+        const textOnly = htmlBody.replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
+        const wordCount = textOnly.split(/\s+/).filter((w) => w.length > 2).length;
+        const hasSpaRoot = /<div[^>]*\s+id=["'](?:root|app|__nuxt|__next|app-root)["'][^>]*>\s*<\/div>/i.test(htmlBody) || /<app-root[^>]*>\s*<\/app-root>/i.test(htmlBody);
+        const hasNoscriptFallback = /<noscript[^>]*>[\s\S]{100,}/i.test(htmlBody);
+        const tinyBody = bodyLen < 3e3 && wordCount < 50;
+        if (hasSpaRoot && wordCount < 100 || tinyBody && !hasNoscriptFallback) {
+          decision = "browser";
         }
-        catch { /* ignore */ }
-        return decision;
+      }
+    } catch {
     }
-    /** Force a domain to use a specific engine (e.g. after observing crawl errors). */
-    set(url, engine) {
-        try {
-            this.cache.set(new URL(url).hostname, engine);
-        }
-        catch { /* ignore */ }
+    try {
+      this.cache.set(new URL(url).hostname, decision);
+    } catch {
     }
-    stats() {
-        const browserDomains = [];
-        const httpDomains = [];
-        for (const [domain, engine] of this.cache) {
-            if (engine === 'browser')
-                browserDomains.push(domain);
-            else
-                httpDomains.push(domain);
-        }
-        return { cached: this.cache.size, browserDomains, httpDomains };
+    return decision;
+  }
+  /** Force a domain to use a specific engine (e.g. after observing crawl errors). */
+  set(url, engine) {
+    try {
+      this.cache.set(new URL(url).hostname, engine);
+    } catch {
     }
-}
-/** Singleton predictor — shared across crawl jobs in the same process. */
-export const renderingPredictor = new RenderingTypePredictor();
-export function chooseCrawlEngines(input) {
-    const engines = [];
-    const reasons = [];
-    const appType = input.appType?.toLowerCase();
-    if (appType === 'android' || appType === 'ios' || appType === 'windows' || appType === 'macos') {
-        return { engines: ['native'], reasons: [`appType=${input.appType}`], confidence: 0.95 };
+  }
+  stats() {
+    const browserDomains = [];
+    const httpDomains = [];
+    for (const [domain, engine] of this.cache) {
+      if (engine === "browser") browserDomains.push(domain);
+      else httpDomains.push(domain);
     }
-    if (input.hasStorageState || input.hasAuthToken) {
-        engines.push('recorded-session');
-        reasons.push('session/token auth available');
-    }
-    engines.push(input.hasProxy ? 'browser-stealth' : 'browser');
-    reasons.push(input.hasProxy ? 'proxy configured; prefer stealth browser path' : 'default browser path');
-    if (input.hasCredentials || input.profile.industry !== 'unknown') {
-        engines.push('stagehand');
-        reasons.push(input.hasCredentials ? 'credentials available for AI login/navigation' : `domain profile=${input.profile.industry}`);
-    }
-    const hardUi = input.profile.industry === 'legacy-enterprise' || /captcha|blocked|canvas|iframe|bot/i.test(input.previousFailureReason ?? '');
-    if (input.hasSkyvern && hardUi) {
-        engines.push('vision');
-        reasons.push('hard UI or previous block detected; vision fallback enabled');
-    }
-    if (engines.length === 1) {
-        engines.unshift('http');
-        reasons.push('HTTP/static extraction can be attempted before browser escalation');
-    }
-    return {
-        engines: Array.from(new Set(engines)),
-        reasons,
-        confidence: input.profile.confidence,
-    };
+    return { cached: this.cache.size, browserDomains, httpDomains };
+  }
+}
+const renderingPredictor = new RenderingTypePredictor();
+function chooseCrawlEngines(input) {
+  const engines = [];
+  const reasons = [];
+  const appType = input.appType?.toLowerCase();
+  if (appType === "android" || appType === "ios" || appType === "windows" || appType === "macos") {
+    return { engines: ["native"], reasons: [`appType=${input.appType}`], confidence: 0.95 };
+  }
+  if (input.hasStorageState || input.hasAuthToken) {
+    engines.push("recorded-session");
+    reasons.push("session/token auth available");
+  }
+  engines.push(input.hasProxy ? "browser-stealth" : "browser");
+  reasons.push(input.hasProxy ? "proxy configured; prefer stealth browser path" : "default browser path");
+  if (input.hasCredentials || input.profile.industry !== "unknown") {
+    engines.push("stagehand");
+    reasons.push(input.hasCredentials ? "credentials available for AI login/navigation" : `domain profile=${input.profile.industry}`);
+  }
+  const hardUi = input.profile.industry === "legacy-enterprise" || /captcha|blocked|canvas|iframe|bot/i.test(input.previousFailureReason ?? "");
+  if (input.hasSkyvern && hardUi) {
+    engines.push("vision");
+    reasons.push("hard UI or previous block detected; vision fallback enabled");
+  }
+  if (engines.length === 1) {
+    engines.unshift("http");
+    reasons.push("HTTP/static extraction can be attempted before browser escalation");
+  }
+  return {
+    engines: Array.from(new Set(engines)),
+    reasons,
+    confidence: input.profile.confidence
+  };
 }
+export {
+  RenderingTypePredictor,
+  chooseCrawlEngines,
+  renderingPredictor
+};
+//# sourceMappingURL=engine-router.js.map