@detiq/crawler 1.1.7 → 1.1.9

package/dist/skyvern.js

@@ -1,129 +1,154 @@
-async function skyvernNavigate(startUrl, navigationGoals, tenantConfig, triggerReason) {
-  const apiUrl = tenantConfig?.apiUrl || process.env.SKYVERN_API_URL;
-  const apiKey = tenantConfig?.apiKey || process.env.SKYVERN_API_KEY;
-  if (!apiUrl || !apiKey) {
-    console.warn("[skyvern] skyvernNavigate: no API config \u2014 skipping post-login vision navigation");
-    return { screenshotUrls: [], visitedUrls: [], status: "failed" };
-  }
-  const visitedUrls = [];
-  const screenshotUrls = [];
-  const skyvernSteps = [];
-  let lastTaskId;
-  let totalDurationMs = 0;
-  for (const goal of navigationGoals) {
-    try {
-      const taskStart = Date.now();
-      const res = await fetch(`${apiUrl}/api/v1/tasks`, {
-        method: "POST",
-        headers: { "Content-Type": "application/json", "x-api-key": apiKey },
-        body: JSON.stringify({
-          url: startUrl,
-          navigation_goal: goal,
-          ...tenantConfig?.llmKey ? { llm_key: tenantConfig.llmKey } : process.env.SKYVERN_LLM_KEY ? { llm_key: process.env.SKYVERN_LLM_KEY } : {},
-          ...tenantConfig?.llmApiKey ? { llm_api_key: tenantConfig.llmApiKey } : process.env.OPENAI_API_KEY ? { llm_api_key: process.env.OPENAI_API_KEY } : {}
-        })
-      });
-      if (!res.ok) continue;
-      const data = await res.json();
-      const taskId = data.task_id;
-      if (!taskId) continue;
-      lastTaskId = taskId;
-      let done = false;
-      for (let p = 0; p < 12 && !done; p++) {
-        await new Promise((r) => setTimeout(r, 5e3));
-        const poll = await fetch(`${apiUrl}/api/v1/tasks/${taskId}`, {
-          headers: { "x-api-key": apiKey }
-        }).catch(() => null);
-        if (!poll?.ok) continue;
-        const pollData = await poll.json();
-        if (pollData.status === "completed" || pollData.status === "failed") {
-          done = true;
-          totalDurationMs += Date.now() - taskStart;
-          if (pollData.status === "completed") {
-            if (pollData.output?.url) visitedUrls.push(pollData.output.url);
-            if (pollData.output?.screenshot_url) screenshotUrls.push(pollData.output.screenshot_url);
-          }
-          try {
-            const stepsRes = await fetch(`${apiUrl}/api/v1/tasks/${taskId}/steps`, {
-              headers: { "x-api-key": apiKey }
-            }).catch(() => null);
-            if (stepsRes?.ok) {
-              const stepsData = await stepsRes.json();
-              const stepsList = Array.isArray(stepsData) ? stepsData : stepsData?.steps ?? [];
-              for (const s of stepsList) {
-                if (s.output?.screenshot_url) screenshotUrls.push(s.output.screenshot_url);
-                skyvernSteps.push({
-                  order: s.step_order ?? s.order ?? 0,
-                  action_type: s.action_type ?? s.action ?? "unknown",
-                  element_description: s.output?.element_description,
-                  reasoning: s.output?.llm_response?.action_description ?? s.output?.reasoning,
-                  success: s.status === "completed",
-                  screenshot_url: s.output?.screenshot_url,
-                  error: s.status === "failed" ? s.output?.errors?.[0] ?? "Step failed" : void 0
-                });
-              }
+/**
+ * Skyvern Vision Fallback
+ *
+ * Skyvern combines computer vision + LLM reasoning to navigate sites that
+ * defeat DOM-based approaches: CAPTCHAs, legacy apps, visual-only login flows,
+ * government portals. It's the "break glass" option when Stagehand fails.
+ *
+ * Production setup (Phase 4+):
+ *   1. Self-host: https://github.com/Skyvern-AI/skyvern
+ *   2. Or use cloud: https://app.skyvern.com
+ *   3. Set SKYVERN_API_URL + SKYVERN_API_KEY in .env
+ *
+ * Pricing note: vision tasks cost more per step than DOM-based Stagehand.
+ * Use only when the fast path fails.
+ */
+/**
+ * Gap 5: Post-login navigation via Skyvern vision for apps where DOM analysis
+ * cannot discover screens (canvas, WebGL, legacy Flash-replacement stacks).
+ * Skyvern drives the browser via screenshot + LLM, returning visited page list.
+ */
+export async function skyvernNavigate(startUrl, navigationGoals, tenantConfig, triggerReason) {
+    const apiUrl = tenantConfig?.apiUrl || process.env.SKYVERN_API_URL;
+    const apiKey = tenantConfig?.apiKey || process.env.SKYVERN_API_KEY;
+    if (!apiUrl || !apiKey) {
+        console.warn('[skyvern] skyvernNavigate: no API config — skipping post-login vision navigation');
+        return { screenshotUrls: [], visitedUrls: [], status: 'failed' };
+    }
+    const visitedUrls = [];
+    const screenshotUrls = [];
+    const skyvernSteps = [];
+    let lastTaskId;
+    let totalDurationMs = 0;
+    for (const goal of navigationGoals) {
+        try {
+            const taskStart = Date.now();
+            const res = await fetch(`${apiUrl}/api/v1/tasks`, {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json', 'x-api-key': apiKey },
+                body: JSON.stringify({
+                    url: startUrl,
+                    navigation_goal: goal,
+                    ...(tenantConfig?.llmKey ? { llm_key: tenantConfig.llmKey } : process.env.SKYVERN_LLM_KEY ? { llm_key: process.env.SKYVERN_LLM_KEY } : {}),
+                    ...(tenantConfig?.llmApiKey ? { llm_api_key: tenantConfig.llmApiKey } : process.env.OPENAI_API_KEY ? { llm_api_key: process.env.OPENAI_API_KEY } : {}),
+                }),
+            });
+            if (!res.ok)
+                continue;
+            const data = await res.json();
+            const taskId = data.task_id;
+            if (!taskId)
+                continue;
+            lastTaskId = taskId;
+            // Poll for task completion (max 60s per goal)
+            let done = false;
+            for (let p = 0; p < 12 && !done; p++) {
+                await new Promise((r) => setTimeout(r, 5000));
+                const poll = await fetch(`${apiUrl}/api/v1/tasks/${taskId}`, {
+                    headers: { 'x-api-key': apiKey },
+                }).catch(() => null);
+                if (!poll?.ok)
+                    continue;
+                const pollData = await poll.json();
+                if (pollData.status === 'completed' || pollData.status === 'failed') {
+                    done = true;
+                    totalDurationMs += Date.now() - taskStart;
+                    if (pollData.status === 'completed') {
+                        if (pollData.output?.url)
+                            visitedUrls.push(pollData.output.url);
+                        if (pollData.output?.screenshot_url)
+                            screenshotUrls.push(pollData.output.screenshot_url);
+                    }
+                    // Fetch step-level detail from Skyvern
+                    try {
+                        const stepsRes = await fetch(`${apiUrl}/api/v1/tasks/${taskId}/steps`, {
+                            headers: { 'x-api-key': apiKey },
+                        }).catch(() => null);
+                        if (stepsRes?.ok) {
+                            const stepsData = await stepsRes.json();
+                            const stepsList = Array.isArray(stepsData) ? stepsData : (stepsData?.steps ?? []);
+                            for (const s of stepsList) {
+                                if (s.output?.screenshot_url)
+                                    screenshotUrls.push(s.output.screenshot_url);
+                                skyvernSteps.push({
+                                    order: s.step_order ?? s.order ?? 0,
+                                    action_type: s.action_type ?? s.action ?? 'unknown',
+                                    element_description: s.output?.element_description,
+                                    reasoning: s.output?.llm_response?.action_description ?? s.output?.reasoning,
+                                    success: s.status === 'completed',
+                                    screenshot_url: s.output?.screenshot_url,
+                                    error: s.status === 'failed' ? (s.output?.errors?.[0] ?? 'Step failed') : undefined,
+                                });
+                            }
+                        }
+                    }
+                    catch { /* non-fatal */ }
+                }
             }
-          } catch {
-          }
         }
-      }
-    } catch (err) {
-      console.warn(`[skyvern] Navigation goal failed: ${goal}`, err);
+        catch (err) {
+            console.warn(`[skyvern] Navigation goal failed: ${goal}`, err);
+        }
     }
-  }
-  return {
-    screenshotUrls,
-    visitedUrls,
-    status: visitedUrls.length > 0 ? "success" : screenshotUrls.length > 0 ? "partial" : "failed",
-    steps: skyvernSteps,
-    taskId: lastTaskId,
-    durationMs: totalDurationMs || void 0,
-    triggerReason
-  };
+    return {
+        screenshotUrls,
+        visitedUrls,
+        status: visitedUrls.length > 0 ? 'success' : screenshotUrls.length > 0 ? 'partial' : 'failed',
+        steps: skyvernSteps,
+        taskId: lastTaskId,
+        durationMs: totalDurationMs || undefined,
+        triggerReason,
+    };
 }
-async function skyvernFallback(task, tenantConfig) {
-  const apiUrl = tenantConfig?.apiUrl || process.env.SKYVERN_API_URL;
-  const apiKey = tenantConfig?.apiKey || process.env.SKYVERN_API_KEY;
-  if (!apiUrl || !apiKey) {
-    console.warn(
-      "[skyvern] Vision fallback disabled \u2014 set SKYVERN_API_URL + SKYVERN_API_KEY in .env to enable.",
-      "\n[skyvern] Self-host: https://github.com/Skyvern-AI/skyvern"
-    );
-    return "failed";
-  }
-  try {
-    const res = await fetch(`${apiUrl}/api/v1/tasks`, {
-      method: "POST",
-      headers: { "Content-Type": "application/json", "x-api-key": apiKey },
-      body: JSON.stringify({
-        url: task.url,
-        navigation_goal: task.taskDescription,
-        ...task.credentials && {
-          navigation_payload: {
-            username: task.credentials.username,
-            password: task.credentials.password,
-            ...task.credentials.oneTimeCode && { otp: task.credentials.oneTimeCode }
-          }
-        },
-        ...tenantConfig?.llmKey ? { llm_key: tenantConfig.llmKey } : process.env.SKYVERN_LLM_KEY ? { llm_key: process.env.SKYVERN_LLM_KEY } : {},
-        ...tenantConfig?.llmApiKey ? { llm_api_key: tenantConfig.llmApiKey } : process.env.OPENAI_API_KEY ? { llm_api_key: process.env.OPENAI_API_KEY } : {}
-      })
-    });
-    if (!res.ok) {
-      const body = await res.text().catch(() => "");
-      console.error(`[skyvern] Task creation failed: ${res.status}`, body);
-      return "failed";
+export async function skyvernFallback(task, tenantConfig) {
+    const apiUrl = tenantConfig?.apiUrl || process.env.SKYVERN_API_URL;
+    const apiKey = tenantConfig?.apiKey || process.env.SKYVERN_API_KEY;
+    if (!apiUrl || !apiKey) {
+        console.warn('[skyvern] Vision fallback disabled — set SKYVERN_API_URL + SKYVERN_API_KEY in .env to enable.', '\n[skyvern] Self-host: https://github.com/Skyvern-AI/skyvern');
+        return 'failed';
+    }
+    try {
+        const res = await fetch(`${apiUrl}/api/v1/tasks`, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json', 'x-api-key': apiKey },
+            body: JSON.stringify({
+                url: task.url,
+                navigation_goal: task.taskDescription,
+                ...(task.credentials && {
+                    navigation_payload: {
+                        username: task.credentials.username,
+                        password: task.credentials.password,
+                        ...(task.credentials.oneTimeCode && { otp: task.credentials.oneTimeCode }),
+                    },
+                }),
+                ...(tenantConfig?.llmKey ? { llm_key: tenantConfig.llmKey } : process.env.SKYVERN_LLM_KEY ? { llm_key: process.env.SKYVERN_LLM_KEY } : {}),
+                ...(tenantConfig?.llmApiKey ? { llm_api_key: tenantConfig.llmApiKey } : process.env.OPENAI_API_KEY ? { llm_api_key: process.env.OPENAI_API_KEY } : {}),
+            }),
+        });
+        if (!res.ok) {
+            const body = await res.text().catch(() => '');
+            console.error(`[skyvern] Task creation failed: ${res.status}`, body);
+            return 'failed';
+        }
+        const data = await res.json();
+        console.log(`[skyvern] Task queued: ${data.task_id}. Poll GET /api/v1/tasks/${data.task_id} for completion.`);
+        // Full implementation: poll until status === "completed" | "failed".
+        // For Phase 1 the login already happened in the browser context Skyvern manages,
+        // so returning 'success' here means we can proceed in that browser session.
+        return 'success';
+    }
+    catch (err) {
+        console.error('[skyvern] Error:', err);
+        return 'failed';
     }
-    const data = await res.json();
-    console.log(`[skyvern] Task queued: ${data.task_id}. Poll GET /api/v1/tasks/${data.task_id} for completion.`);
-    return "success";
-  } catch (err) {
-    console.error("[skyvern] Error:", err);
-    return "failed";
-  }
 }
-export {
-  skyvernFallback,
-  skyvernNavigate
-};
-//# sourceMappingURL=skyvern.js.map

package/dist/sqs-queue.js

@@ -1,47 +1,47 @@
-import { SQSClient, SendMessageCommand, ReceiveMessageCommand, DeleteMessageCommand } from "@aws-sdk/client-sqs";
-const sqs = new SQSClient({ region: process.env.AWS_REGION ?? "us-east-1" });
-async function sendJobToSqs(jobId, tenantId, config) {
-  const queueUrl = process.env.SQS_CRAWL_QUEUE_URL;
-  if (!queueUrl) throw new Error("SQS_CRAWL_QUEUE_URL not configured");
-  await sqs.send(new SendMessageCommand({
-    QueueUrl: queueUrl,
-    MessageBody: JSON.stringify({ jobId, tenantId, config })
-  }));
-}
-async function startSqsWorker() {
-  const queueUrl = process.env.SQS_CRAWL_QUEUE_URL;
-  if (!queueUrl) throw new Error("SQS_CRAWL_QUEUE_URL not configured");
-  const { processJob } = await import("./job-queue.js");
-  console.log("[sqs-worker] starting, queue:", queueUrl);
-  while (true) {
-    try {
-      const { Messages = [] } = await sqs.send(new ReceiveMessageCommand({
+import { SQSClient, SendMessageCommand, ReceiveMessageCommand, DeleteMessageCommand } from '@aws-sdk/client-sqs';
+const sqs = new SQSClient({ region: process.env.AWS_REGION ?? 'us-east-1' });
+export async function sendJobToSqs(jobId, tenantId, config) {
+    const queueUrl = process.env.SQS_CRAWL_QUEUE_URL;
+    if (!queueUrl)
+        throw new Error('SQS_CRAWL_QUEUE_URL not configured');
+    await sqs.send(new SendMessageCommand({
         QueueUrl: queueUrl,
-        WaitTimeSeconds: 20,
-        MaxNumberOfMessages: 2
-      }));
-      await Promise.all(Messages.map(async (msg) => {
-        let jobId = "?";
+        MessageBody: JSON.stringify({ jobId, tenantId, config }),
+    }));
+}
+export async function startSqsWorker() {
+    const queueUrl = process.env.SQS_CRAWL_QUEUE_URL;
+    if (!queueUrl)
+        throw new Error('SQS_CRAWL_QUEUE_URL not configured');
+    const { processJob } = await import('./job-queue.js');
+    console.log('[sqs-worker] starting, queue:', queueUrl);
+    while (true) {
         try {
-          const data = JSON.parse(msg.Body);
-          jobId = data.jobId;
-          await processJob(data.jobId, data.config);
-          await sqs.send(new DeleteMessageCommand({
-            QueueUrl: queueUrl,
-            ReceiptHandle: msg.ReceiptHandle
-          }));
-        } catch (err) {
-          console.error(`[sqs-worker] job ${jobId} failed \u2014 leaving for SQS redrive:`, err.message);
+            const { Messages = [] } = await sqs.send(new ReceiveMessageCommand({
+                QueueUrl: queueUrl,
+                WaitTimeSeconds: 20,
+                MaxNumberOfMessages: 2,
+            }));
+            await Promise.all(Messages.map(async (msg) => {
+                let jobId = '?';
+                try {
+                    const data = JSON.parse(msg.Body);
+                    jobId = data.jobId;
+                    await processJob(data.jobId, data.config);
+                    await sqs.send(new DeleteMessageCommand({
+                        QueueUrl: queueUrl,
+                        ReceiptHandle: msg.ReceiptHandle,
+                    }));
+                }
+                catch (err) {
+                    console.error(`[sqs-worker] job ${jobId} failed — leaving for SQS redrive:`, err.message);
+                    // Do NOT delete — SQS redrive policy + DLQ handles retry
+                }
+            }));
+        }
+        catch (err) {
+            console.error('[sqs-worker] receive error:', err.message);
+            await new Promise((r) => setTimeout(r, 2000));
         }
-      }));
-    } catch (err) {
-      console.error("[sqs-worker] receive error:", err.message);
-      await new Promise((r) => setTimeout(r, 2e3));
     }
-  }
 }
-export {
-  sendJobToSqs,
-  startSqsWorker
-};
-//# sourceMappingURL=sqs-queue.js.map

package/dist/sso-token.js

@@ -1,76 +1,92 @@
-async function getSsoToken(config, username, password) {
-  let tokenUrl;
-  const params = {
-    grant_type: "password",
-    username,
-    password,
-    scope: config.scope ?? "openid profile",
-    client_id: config.clientId
-  };
-  if (config.clientSecret) {
-    params.client_secret = config.clientSecret;
-  }
-  switch (config.provider) {
-    case "okta":
-      tokenUrl = `https://${config.domain}/oauth2/v1/token`;
-      break;
-    case "azure_ad":
-      tokenUrl = `https://login.microsoftonline.com/${config.domain}/oauth2/v2.0/token`;
-      break;
-    case "generic_oidc":
-      tokenUrl = config.domain.startsWith("http") ? config.domain : `https://${config.domain}/oauth2/token`;
-      break;
-    default:
-      throw new Error(`Unknown SSO provider: ${config.provider}`);
-  }
-  const res = await fetch(tokenUrl, {
-    method: "POST",
-    headers: { "Content-Type": "application/x-www-form-urlencoded" },
-    body: new URLSearchParams(params).toString()
-  });
-  if (!res.ok) {
-    const body = await res.text().catch(() => res.statusText);
-    throw new Error(`SSO token request failed (${res.status}): ${body}`);
-  }
-  const data = await res.json();
-  if (data.error) throw new Error(`SSO error: ${data.error} \u2014 ${data.error_description}`);
-  if (!data.access_token) throw new Error("SSO response missing access_token");
-  return data.access_token;
+/**
+ * SSO token retrieval for Okta and Azure AD.
+ * Uses OAuth2 ROPC (Resource Owner Password Credentials) flow to get access tokens
+ * without browser-based login.
+ *
+ * Note: ROPC is deprecated in OAuth 2.1 but still supported by Okta and Azure AD
+ * for service accounts and automated testing scenarios.
+ */
+/**
+ * Get an access token using ROPC flow (username + password → token).
+ * Returns the access token string.
+ */
+export async function getSsoToken(config, username, password) {
+    let tokenUrl;
+    const params = {
+        grant_type: 'password',
+        username,
+        password,
+        scope: config.scope ?? 'openid profile',
+        client_id: config.clientId,
+    };
+    if (config.clientSecret) {
+        params.client_secret = config.clientSecret;
+    }
+    switch (config.provider) {
+        case 'okta':
+            tokenUrl = `https://${config.domain}/oauth2/v1/token`;
+            break;
+        case 'azure_ad':
+            // domain here is the tenant ID (e.g. XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX or 'mycompany.onmicrosoft.com')
+            tokenUrl = `https://login.microsoftonline.com/${config.domain}/oauth2/v2.0/token`;
+            break;
+        case 'generic_oidc':
+            tokenUrl = config.domain.startsWith('http') ? config.domain : `https://${config.domain}/oauth2/token`;
+            break;
+        default:
+            throw new Error(`Unknown SSO provider: ${config.provider}`);
+    }
+    const res = await fetch(tokenUrl, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
+        body: new URLSearchParams(params).toString(),
+    });
+    if (!res.ok) {
+        const body = await res.text().catch(() => res.statusText);
+        throw new Error(`SSO token request failed (${res.status}): ${body}`);
+    }
+    const data = await res.json();
+    if (data.error)
+        throw new Error(`SSO error: ${data.error} — ${data.error_description}`);
+    if (!data.access_token)
+        throw new Error('SSO response missing access_token');
+    return data.access_token;
 }
-async function getSsoClientCredentialsToken(config) {
-  if (!config.clientSecret) throw new Error("clientSecret required for client credentials flow");
-  let tokenUrl;
-  switch (config.provider) {
-    case "okta":
-      tokenUrl = `https://${config.domain}/oauth2/v1/token`;
-      break;
-    case "azure_ad":
-      tokenUrl = `https://login.microsoftonline.com/${config.domain}/oauth2/v2.0/token`;
-      break;
-    default:
-      tokenUrl = `https://${config.domain}/oauth2/token`;
-  }
-  const res = await fetch(tokenUrl, {
-    method: "POST",
-    headers: { "Content-Type": "application/x-www-form-urlencoded" },
-    body: new URLSearchParams({
-      grant_type: "client_credentials",
-      client_id: config.clientId,
-      client_secret: config.clientSecret,
-      scope: config.scope ?? "openid profile"
-    }).toString()
-  });
-  if (!res.ok) {
-    const body = await res.text().catch(() => res.statusText);
-    throw new Error(`SSO client credentials failed (${res.status}): ${body}`);
-  }
-  const data = await res.json();
-  if (data.error) throw new Error(`SSO error: ${data.error} \u2014 ${data.error_description}`);
-  if (!data.access_token) throw new Error("SSO response missing access_token");
-  return data.access_token;
+/**
+ * Get a client credentials token (no user — for service accounts).
+ */
+export async function getSsoClientCredentialsToken(config) {
+    if (!config.clientSecret)
+        throw new Error('clientSecret required for client credentials flow');
+    let tokenUrl;
+    switch (config.provider) {
+        case 'okta':
+            tokenUrl = `https://${config.domain}/oauth2/v1/token`;
+            break;
+        case 'azure_ad':
+            tokenUrl = `https://login.microsoftonline.com/${config.domain}/oauth2/v2.0/token`;
+            break;
+        default:
+            tokenUrl = `https://${config.domain}/oauth2/token`;
+    }
+    const res = await fetch(tokenUrl, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
+        body: new URLSearchParams({
+            grant_type: 'client_credentials',
+            client_id: config.clientId,
+            client_secret: config.clientSecret,
+            scope: config.scope ?? 'openid profile',
+        }).toString(),
+    });
+    if (!res.ok) {
+        const body = await res.text().catch(() => res.statusText);
+        throw new Error(`SSO client credentials failed (${res.status}): ${body}`);
+    }
+    const data = await res.json();
+    if (data.error)
+        throw new Error(`SSO error: ${data.error} — ${data.error_description}`);
+    if (!data.access_token)
+        throw new Error('SSO response missing access_token');
+    return data.access_token;
 }
-export {
-  getSsoClientCredentialsToken,
-  getSsoToken
-};
-//# sourceMappingURL=sso-token.js.map