@detiq/crawler 1.1.9 → 1.1.11

package/dist/skyvern.js

@@ -1,154 +1,129 @@
-/**
- * Skyvern Vision Fallback
- *
- * Skyvern combines computer vision + LLM reasoning to navigate sites that
- * defeat DOM-based approaches: CAPTCHAs, legacy apps, visual-only login flows,
- * government portals. It's the "break glass" option when Stagehand fails.
- *
- * Production setup (Phase 4+):
- *   1. Self-host: https://github.com/Skyvern-AI/skyvern
- *   2. Or use cloud: https://app.skyvern.com
- *   3. Set SKYVERN_API_URL + SKYVERN_API_KEY in .env
- *
- * Pricing note: vision tasks cost more per step than DOM-based Stagehand.
- * Use only when the fast path fails.
- */
-/**
- * Gap 5: Post-login navigation via Skyvern vision for apps where DOM analysis
- * cannot discover screens (canvas, WebGL, legacy Flash-replacement stacks).
- * Skyvern drives the browser via screenshot + LLM, returning visited page list.
- */
-export async function skyvernNavigate(startUrl, navigationGoals, tenantConfig, triggerReason) {
-    const apiUrl = tenantConfig?.apiUrl || process.env.SKYVERN_API_URL;
-    const apiKey = tenantConfig?.apiKey || process.env.SKYVERN_API_KEY;
-    if (!apiUrl || !apiKey) {
-        console.warn('[skyvern] skyvernNavigate: no API config — skipping post-login vision navigation');
-        return { screenshotUrls: [], visitedUrls: [], status: 'failed' };
-    }
-    const visitedUrls = [];
-    const screenshotUrls = [];
-    const skyvernSteps = [];
-    let lastTaskId;
-    let totalDurationMs = 0;
-    for (const goal of navigationGoals) {
-        try {
-            const taskStart = Date.now();
-            const res = await fetch(`${apiUrl}/api/v1/tasks`, {
-                method: 'POST',
-                headers: { 'Content-Type': 'application/json', 'x-api-key': apiKey },
-                body: JSON.stringify({
-                    url: startUrl,
-                    navigation_goal: goal,
-                    ...(tenantConfig?.llmKey ? { llm_key: tenantConfig.llmKey } : process.env.SKYVERN_LLM_KEY ? { llm_key: process.env.SKYVERN_LLM_KEY } : {}),
-                    ...(tenantConfig?.llmApiKey ? { llm_api_key: tenantConfig.llmApiKey } : process.env.OPENAI_API_KEY ? { llm_api_key: process.env.OPENAI_API_KEY } : {}),
-                }),
-            });
-            if (!res.ok)
-                continue;
-            const data = await res.json();
-            const taskId = data.task_id;
-            if (!taskId)
-                continue;
-            lastTaskId = taskId;
-            // Poll for task completion (max 60s per goal)
-            let done = false;
-            for (let p = 0; p < 12 && !done; p++) {
-                await new Promise((r) => setTimeout(r, 5000));
-                const poll = await fetch(`${apiUrl}/api/v1/tasks/${taskId}`, {
-                    headers: { 'x-api-key': apiKey },
-                }).catch(() => null);
-                if (!poll?.ok)
-                    continue;
-                const pollData = await poll.json();
-                if (pollData.status === 'completed' || pollData.status === 'failed') {
-                    done = true;
-                    totalDurationMs += Date.now() - taskStart;
-                    if (pollData.status === 'completed') {
-                        if (pollData.output?.url)
-                            visitedUrls.push(pollData.output.url);
-                        if (pollData.output?.screenshot_url)
-                            screenshotUrls.push(pollData.output.screenshot_url);
-                    }
-                    // Fetch step-level detail from Skyvern
-                    try {
-                        const stepsRes = await fetch(`${apiUrl}/api/v1/tasks/${taskId}/steps`, {
-                            headers: { 'x-api-key': apiKey },
-                        }).catch(() => null);
-                        if (stepsRes?.ok) {
-                            const stepsData = await stepsRes.json();
-                            const stepsList = Array.isArray(stepsData) ? stepsData : (stepsData?.steps ?? []);
-                            for (const s of stepsList) {
-                                if (s.output?.screenshot_url)
-                                    screenshotUrls.push(s.output.screenshot_url);
-                                skyvernSteps.push({
-                                    order: s.step_order ?? s.order ?? 0,
-                                    action_type: s.action_type ?? s.action ?? 'unknown',
-                                    element_description: s.output?.element_description,
-                                    reasoning: s.output?.llm_response?.action_description ?? s.output?.reasoning,
-                                    success: s.status === 'completed',
-                                    screenshot_url: s.output?.screenshot_url,
-                                    error: s.status === 'failed' ? (s.output?.errors?.[0] ?? 'Step failed') : undefined,
-                                });
-                            }
-                        }
-                    }
-                    catch { /* non-fatal */ }
-                }
+async function skyvernNavigate(startUrl, navigationGoals, tenantConfig, triggerReason) {
+  const apiUrl = tenantConfig?.apiUrl || process.env.SKYVERN_API_URL;
+  const apiKey = tenantConfig?.apiKey || process.env.SKYVERN_API_KEY;
+  if (!apiUrl || !apiKey) {
+    console.warn("[skyvern] skyvernNavigate: no API config \u2014 skipping post-login vision navigation");
+    return { screenshotUrls: [], visitedUrls: [], status: "failed" };
+  }
+  const visitedUrls = [];
+  const screenshotUrls = [];
+  const skyvernSteps = [];
+  let lastTaskId;
+  let totalDurationMs = 0;
+  for (const goal of navigationGoals) {
+    try {
+      const taskStart = Date.now();
+      const res = await fetch(`${apiUrl}/api/v1/tasks`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json", "x-api-key": apiKey },
+        body: JSON.stringify({
+          url: startUrl,
+          navigation_goal: goal,
+          ...tenantConfig?.llmKey ? { llm_key: tenantConfig.llmKey } : process.env.SKYVERN_LLM_KEY ? { llm_key: process.env.SKYVERN_LLM_KEY } : {},
+          ...tenantConfig?.llmApiKey ? { llm_api_key: tenantConfig.llmApiKey } : process.env.OPENAI_API_KEY ? { llm_api_key: process.env.OPENAI_API_KEY } : {}
+        })
+      });
+      if (!res.ok) continue;
+      const data = await res.json();
+      const taskId = data.task_id;
+      if (!taskId) continue;
+      lastTaskId = taskId;
+      let done = false;
+      for (let p = 0; p < 12 && !done; p++) {
+        await new Promise((r) => setTimeout(r, 5e3));
+        const poll = await fetch(`${apiUrl}/api/v1/tasks/${taskId}`, {
+          headers: { "x-api-key": apiKey }
+        }).catch(() => null);
+        if (!poll?.ok) continue;
+        const pollData = await poll.json();
+        if (pollData.status === "completed" || pollData.status === "failed") {
+          done = true;
+          totalDurationMs += Date.now() - taskStart;
+          if (pollData.status === "completed") {
+            if (pollData.output?.url) visitedUrls.push(pollData.output.url);
+            if (pollData.output?.screenshot_url) screenshotUrls.push(pollData.output.screenshot_url);
+          }
+          try {
+            const stepsRes = await fetch(`${apiUrl}/api/v1/tasks/${taskId}/steps`, {
+              headers: { "x-api-key": apiKey }
+            }).catch(() => null);
+            if (stepsRes?.ok) {
+              const stepsData = await stepsRes.json();
+              const stepsList = Array.isArray(stepsData) ? stepsData : stepsData?.steps ?? [];
+              for (const s of stepsList) {
+                if (s.output?.screenshot_url) screenshotUrls.push(s.output.screenshot_url);
+                skyvernSteps.push({
+                  order: s.step_order ?? s.order ?? 0,
+                  action_type: s.action_type ?? s.action ?? "unknown",
+                  element_description: s.output?.element_description,
+                  reasoning: s.output?.llm_response?.action_description ?? s.output?.reasoning,
+                  success: s.status === "completed",
+                  screenshot_url: s.output?.screenshot_url,
+                  error: s.status === "failed" ? s.output?.errors?.[0] ?? "Step failed" : void 0
+                });
+              }
             }
+          } catch {
+          }
         }
-        catch (err) {
-            console.warn(`[skyvern] Navigation goal failed: ${goal}`, err);
-        }
+      }
+    } catch (err) {
+      console.warn(`[skyvern] Navigation goal failed: ${goal}`, err);
     }
-    return {
-        screenshotUrls,
-        visitedUrls,
-        status: visitedUrls.length > 0 ? 'success' : screenshotUrls.length > 0 ? 'partial' : 'failed',
-        steps: skyvernSteps,
-        taskId: lastTaskId,
-        durationMs: totalDurationMs || undefined,
-        triggerReason,
-    };
+  }
+  return {
+    screenshotUrls,
+    visitedUrls,
+    status: visitedUrls.length > 0 ? "success" : screenshotUrls.length > 0 ? "partial" : "failed",
+    steps: skyvernSteps,
+    taskId: lastTaskId,
+    durationMs: totalDurationMs || void 0,
+    triggerReason
+  };
 }
-export async function skyvernFallback(task, tenantConfig) {
-    const apiUrl = tenantConfig?.apiUrl || process.env.SKYVERN_API_URL;
-    const apiKey = tenantConfig?.apiKey || process.env.SKYVERN_API_KEY;
-    if (!apiUrl || !apiKey) {
-        console.warn('[skyvern] Vision fallback disabled — set SKYVERN_API_URL + SKYVERN_API_KEY in .env to enable.', '\n[skyvern] Self-host: https://github.com/Skyvern-AI/skyvern');
-        return 'failed';
-    }
-    try {
-        const res = await fetch(`${apiUrl}/api/v1/tasks`, {
-            method: 'POST',
-            headers: { 'Content-Type': 'application/json', 'x-api-key': apiKey },
-            body: JSON.stringify({
-                url: task.url,
-                navigation_goal: task.taskDescription,
-                ...(task.credentials && {
-                    navigation_payload: {
-                        username: task.credentials.username,
-                        password: task.credentials.password,
-                        ...(task.credentials.oneTimeCode && { otp: task.credentials.oneTimeCode }),
-                    },
-                }),
-                ...(tenantConfig?.llmKey ? { llm_key: tenantConfig.llmKey } : process.env.SKYVERN_LLM_KEY ? { llm_key: process.env.SKYVERN_LLM_KEY } : {}),
-                ...(tenantConfig?.llmApiKey ? { llm_api_key: tenantConfig.llmApiKey } : process.env.OPENAI_API_KEY ? { llm_api_key: process.env.OPENAI_API_KEY } : {}),
-            }),
-        });
-        if (!res.ok) {
-            const body = await res.text().catch(() => '');
-            console.error(`[skyvern] Task creation failed: ${res.status}`, body);
-            return 'failed';
-        }
-        const data = await res.json();
-        console.log(`[skyvern] Task queued: ${data.task_id}. Poll GET /api/v1/tasks/${data.task_id} for completion.`);
-        // Full implementation: poll until status === "completed" | "failed".
-        // For Phase 1 the login already happened in the browser context Skyvern manages,
-        // so returning 'success' here means we can proceed in that browser session.
-        return 'success';
-    }
-    catch (err) {
-        console.error('[skyvern] Error:', err);
-        return 'failed';
+async function skyvernFallback(task, tenantConfig) {
+  const apiUrl = tenantConfig?.apiUrl || process.env.SKYVERN_API_URL;
+  const apiKey = tenantConfig?.apiKey || process.env.SKYVERN_API_KEY;
+  if (!apiUrl || !apiKey) {
+    console.warn(
+      "[skyvern] Vision fallback disabled \u2014 set SKYVERN_API_URL + SKYVERN_API_KEY in .env to enable.",
+      "\n[skyvern] Self-host: https://github.com/Skyvern-AI/skyvern"
+    );
+    return "failed";
+  }
+  try {
+    const res = await fetch(`${apiUrl}/api/v1/tasks`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json", "x-api-key": apiKey },
+      body: JSON.stringify({
+        url: task.url,
+        navigation_goal: task.taskDescription,
+        ...task.credentials && {
+          navigation_payload: {
+            username: task.credentials.username,
+            password: task.credentials.password,
+            ...task.credentials.oneTimeCode && { otp: task.credentials.oneTimeCode }
+          }
+        },
+        ...tenantConfig?.llmKey ? { llm_key: tenantConfig.llmKey } : process.env.SKYVERN_LLM_KEY ? { llm_key: process.env.SKYVERN_LLM_KEY } : {},
+        ...tenantConfig?.llmApiKey ? { llm_api_key: tenantConfig.llmApiKey } : process.env.OPENAI_API_KEY ? { llm_api_key: process.env.OPENAI_API_KEY } : {}
+      })
+    });
+    if (!res.ok) {
+      const body = await res.text().catch(() => "");
+      console.error(`[skyvern] Task creation failed: ${res.status}`, body);
+      return "failed";
     }
+    const data = await res.json();
+    console.log(`[skyvern] Task queued: ${data.task_id}. Poll GET /api/v1/tasks/${data.task_id} for completion.`);
+    return "success";
+  } catch (err) {
+    console.error("[skyvern] Error:", err);
+    return "failed";
+  }
 }
+export {
+  skyvernFallback,
+  skyvernNavigate
+};
+//# sourceMappingURL=skyvern.js.map

package/dist/sqs-queue.js

@@ -1,47 +1,47 @@
-import { SQSClient, SendMessageCommand, ReceiveMessageCommand, DeleteMessageCommand } from '@aws-sdk/client-sqs';
-const sqs = new SQSClient({ region: process.env.AWS_REGION ?? 'us-east-1' });
-export async function sendJobToSqs(jobId, tenantId, config) {
-    const queueUrl = process.env.SQS_CRAWL_QUEUE_URL;
-    if (!queueUrl)
-        throw new Error('SQS_CRAWL_QUEUE_URL not configured');
-    await sqs.send(new SendMessageCommand({
-        QueueUrl: queueUrl,
-        MessageBody: JSON.stringify({ jobId, tenantId, config }),
-    }));
+import { SQSClient, SendMessageCommand, ReceiveMessageCommand, DeleteMessageCommand } from "@aws-sdk/client-sqs";
+const sqs = new SQSClient({ region: process.env.AWS_REGION ?? "us-east-1" });
+async function sendJobToSqs(jobId, tenantId, config) {
+  const queueUrl = process.env.SQS_CRAWL_QUEUE_URL;
+  if (!queueUrl) throw new Error("SQS_CRAWL_QUEUE_URL not configured");
+  await sqs.send(new SendMessageCommand({
+    QueueUrl: queueUrl,
+    MessageBody: JSON.stringify({ jobId, tenantId, config })
+  }));
 }
-export async function startSqsWorker() {
-    const queueUrl = process.env.SQS_CRAWL_QUEUE_URL;
-    if (!queueUrl)
-        throw new Error('SQS_CRAWL_QUEUE_URL not configured');
-    const { processJob } = await import('./job-queue.js');
-    console.log('[sqs-worker] starting, queue:', queueUrl);
-    while (true) {
+async function startSqsWorker() {
+  const queueUrl = process.env.SQS_CRAWL_QUEUE_URL;
+  if (!queueUrl) throw new Error("SQS_CRAWL_QUEUE_URL not configured");
+  const { processJob } = await import("./job-queue.js");
+  console.log("[sqs-worker] starting, queue:", queueUrl);
+  while (true) {
+    try {
+      const { Messages = [] } = await sqs.send(new ReceiveMessageCommand({
+        QueueUrl: queueUrl,
+        WaitTimeSeconds: 20,
+        MaxNumberOfMessages: 2
+      }));
+      await Promise.all(Messages.map(async (msg) => {
+        let jobId = "?";
         try {
-            const { Messages = [] } = await sqs.send(new ReceiveMessageCommand({
-                QueueUrl: queueUrl,
-                WaitTimeSeconds: 20,
-                MaxNumberOfMessages: 2,
-            }));
-            await Promise.all(Messages.map(async (msg) => {
-                let jobId = '?';
-                try {
-                    const data = JSON.parse(msg.Body);
-                    jobId = data.jobId;
-                    await processJob(data.jobId, data.config);
-                    await sqs.send(new DeleteMessageCommand({
-                        QueueUrl: queueUrl,
-                        ReceiptHandle: msg.ReceiptHandle,
-                    }));
-                }
-                catch (err) {
-                    console.error(`[sqs-worker] job ${jobId} failed — leaving for SQS redrive:`, err.message);
-                    // Do NOT delete — SQS redrive policy + DLQ handles retry
-                }
-            }));
-        }
-        catch (err) {
-            console.error('[sqs-worker] receive error:', err.message);
-            await new Promise((r) => setTimeout(r, 2000));
+          const data = JSON.parse(msg.Body);
+          jobId = data.jobId;
+          await processJob(data.jobId, data.config);
+          await sqs.send(new DeleteMessageCommand({
+            QueueUrl: queueUrl,
+            ReceiptHandle: msg.ReceiptHandle
+          }));
+        } catch (err) {
+          console.error(`[sqs-worker] job ${jobId} failed \u2014 leaving for SQS redrive:`, err.message);
         }
+      }));
+    } catch (err) {
+      console.error("[sqs-worker] receive error:", err.message);
+      await new Promise((r) => setTimeout(r, 2e3));
     }
+  }
 }
+export {
+  sendJobToSqs,
+  startSqsWorker
+};
+//# sourceMappingURL=sqs-queue.js.map

package/dist/sso-token.js

@@ -1,92 +1,76 @@
-/**
- * SSO token retrieval for Okta and Azure AD.
- * Uses OAuth2 ROPC (Resource Owner Password Credentials) flow to get access tokens
- * without browser-based login.
- *
- * Note: ROPC is deprecated in OAuth 2.1 but still supported by Okta and Azure AD
- * for service accounts and automated testing scenarios.
- */
-/**
- * Get an access token using ROPC flow (username + password → token).
- * Returns the access token string.
- */
-export async function getSsoToken(config, username, password) {
-    let tokenUrl;
-    const params = {
-        grant_type: 'password',
-        username,
-        password,
-        scope: config.scope ?? 'openid profile',
-        client_id: config.clientId,
-    };
-    if (config.clientSecret) {
-        params.client_secret = config.clientSecret;
-    }
-    switch (config.provider) {
-        case 'okta':
-            tokenUrl = `https://${config.domain}/oauth2/v1/token`;
-            break;
-        case 'azure_ad':
-            // domain here is the tenant ID (e.g. XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX or 'mycompany.onmicrosoft.com')
-            tokenUrl = `https://login.microsoftonline.com/${config.domain}/oauth2/v2.0/token`;
-            break;
-        case 'generic_oidc':
-            tokenUrl = config.domain.startsWith('http') ? config.domain : `https://${config.domain}/oauth2/token`;
-            break;
-        default:
-            throw new Error(`Unknown SSO provider: ${config.provider}`);
-    }
-    const res = await fetch(tokenUrl, {
-        method: 'POST',
-        headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
-        body: new URLSearchParams(params).toString(),
-    });
-    if (!res.ok) {
-        const body = await res.text().catch(() => res.statusText);
-        throw new Error(`SSO token request failed (${res.status}): ${body}`);
-    }
-    const data = await res.json();
-    if (data.error)
-        throw new Error(`SSO error: ${data.error} — ${data.error_description}`);
-    if (!data.access_token)
-        throw new Error('SSO response missing access_token');
-    return data.access_token;
+async function getSsoToken(config, username, password) {
+  let tokenUrl;
+  const params = {
+    grant_type: "password",
+    username,
+    password,
+    scope: config.scope ?? "openid profile",
+    client_id: config.clientId
+  };
+  if (config.clientSecret) {
+    params.client_secret = config.clientSecret;
+  }
+  switch (config.provider) {
+    case "okta":
+      tokenUrl = `https://${config.domain}/oauth2/v1/token`;
+      break;
+    case "azure_ad":
+      tokenUrl = `https://login.microsoftonline.com/${config.domain}/oauth2/v2.0/token`;
+      break;
+    case "generic_oidc":
+      tokenUrl = config.domain.startsWith("http") ? config.domain : `https://${config.domain}/oauth2/token`;
+      break;
+    default:
+      throw new Error(`Unknown SSO provider: ${config.provider}`);
+  }
+  const res = await fetch(tokenUrl, {
+    method: "POST",
+    headers: { "Content-Type": "application/x-www-form-urlencoded" },
+    body: new URLSearchParams(params).toString()
+  });
+  if (!res.ok) {
+    const body = await res.text().catch(() => res.statusText);
+    throw new Error(`SSO token request failed (${res.status}): ${body}`);
+  }
+  const data = await res.json();
+  if (data.error) throw new Error(`SSO error: ${data.error} \u2014 ${data.error_description}`);
+  if (!data.access_token) throw new Error("SSO response missing access_token");
+  return data.access_token;
 }
-/**
- * Get a client credentials token (no user — for service accounts).
- */
-export async function getSsoClientCredentialsToken(config) {
-    if (!config.clientSecret)
-        throw new Error('clientSecret required for client credentials flow');
-    let tokenUrl;
-    switch (config.provider) {
-        case 'okta':
-            tokenUrl = `https://${config.domain}/oauth2/v1/token`;
-            break;
-        case 'azure_ad':
-            tokenUrl = `https://login.microsoftonline.com/${config.domain}/oauth2/v2.0/token`;
-            break;
-        default:
-            tokenUrl = `https://${config.domain}/oauth2/token`;
-    }
-    const res = await fetch(tokenUrl, {
-        method: 'POST',
-        headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
-        body: new URLSearchParams({
-            grant_type: 'client_credentials',
-            client_id: config.clientId,
-            client_secret: config.clientSecret,
-            scope: config.scope ?? 'openid profile',
-        }).toString(),
-    });
-    if (!res.ok) {
-        const body = await res.text().catch(() => res.statusText);
-        throw new Error(`SSO client credentials failed (${res.status}): ${body}`);
-    }
-    const data = await res.json();
-    if (data.error)
-        throw new Error(`SSO error: ${data.error} — ${data.error_description}`);
-    if (!data.access_token)
-        throw new Error('SSO response missing access_token');
-    return data.access_token;
+async function getSsoClientCredentialsToken(config) {
+  if (!config.clientSecret) throw new Error("clientSecret required for client credentials flow");
+  let tokenUrl;
+  switch (config.provider) {
+    case "okta":
+      tokenUrl = `https://${config.domain}/oauth2/v1/token`;
+      break;
+    case "azure_ad":
+      tokenUrl = `https://login.microsoftonline.com/${config.domain}/oauth2/v2.0/token`;
+      break;
+    default:
+      tokenUrl = `https://${config.domain}/oauth2/token`;
+  }
+  const res = await fetch(tokenUrl, {
+    method: "POST",
+    headers: { "Content-Type": "application/x-www-form-urlencoded" },
+    body: new URLSearchParams({
+      grant_type: "client_credentials",
+      client_id: config.clientId,
+      client_secret: config.clientSecret,
+      scope: config.scope ?? "openid profile"
+    }).toString()
+  });
+  if (!res.ok) {
+    const body = await res.text().catch(() => res.statusText);
+    throw new Error(`SSO client credentials failed (${res.status}): ${body}`);
+  }
+  const data = await res.json();
+  if (data.error) throw new Error(`SSO error: ${data.error} \u2014 ${data.error_description}`);
+  if (!data.access_token) throw new Error("SSO response missing access_token");
+  return data.access_token;
 }
+export {
+  getSsoClientCredentialsToken,
+  getSsoToken
+};
+//# sourceMappingURL=sso-token.js.map