@taodigital/crawler 0.0.0-stage → 1.2.1

package/src/sitemap.ts

@@ -0,0 +1,233 @@
+/**
+ * Sitemap.xml discovery — seeds the crawl queue from a project's sitemap
+ * instead of (or alongside) plain link-following, matching crawl4AI's
+ * sitemap-seeding capability. Best-effort throughout: most sites don't have
+ * a sitemap, or it 404s, or it's malformed — none of that should ever fail
+ * a crawl, it just means sitemap seeding contributes nothing this run.
+ */
+import { XMLParser } from 'fast-xml-parser';
+import { ProxyAgent } from 'undici';
+import { assertCrawlUrlSafe } from '@taodigital/core/dist/ssrf-guard.js';
+import { logger } from './logger.js';
+
+const FETCH_TIMEOUT_MS = 10_000;
+const MAX_SITEMAP_INDEX_DEPTH = 3; // sitemap index -> sub-sitemaps -> (no deeper nesting expected)
+const MAX_SUB_SITEMAPS = 50; // bound fan-out from a single sitemap index
+
+async function getSitemapMaxUrls(): Promise<number> {
+  const { getSystemConfig } = await import('@taodigital/core');
+  const cfg = await getSystemConfig('crawl_tuning');
+  return parseInt(String(cfg?.sitemapMaxUrls ?? '5000'), 10);
+}
+
+function toArray<T>(x: T | T[] | undefined | null): T[] {
+  if (x == null) return [];
+  return Array.isArray(x) ? x : [x];
+}
+
+/** Registrable-host key: lowercased hostname with a leading `www.` stripped. */
+function hostKey(hostname: string): string {
+  return hostname.replace(/^www\./i, '').toLowerCase();
+}
+
+/**
+ * Same-site check that tolerates www/non-www and http/https differences — a sitemap on
+ * `vercel.com` is valid for a project pointed at `www.vercel.com` (and vice versa).
+ */
+export function sameSite(candidateUrl: string, siteOrigin: string): boolean {
+  try {
+    const originHost = /^https?:\/\//i.test(siteOrigin) ? new URL(siteOrigin).hostname : siteOrigin;
+    return hostKey(new URL(candidateUrl).hostname) === hostKey(originHost);
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Pulls page URLs and sub-sitemap URLs out of an already-XML-parsed sitemap
+ * document (either a <urlset> or a <sitemapindex>), filtered to same-origin
+ * and deduped. Pure function — no I/O — for direct unit testing without a
+ * real HTTP fetch.
+ */
+export function extractUrlsFromParsedSitemap(parsed: any, origin: string, maxUrls = 5000): { pageUrls: string[]; subSitemapUrls: string[] } {
+  const rawPageUrls: string[] = [];
+  let subSitemapUrls: string[] = [];
+
+  if (parsed?.sitemapindex) {
+    subSitemapUrls = toArray(parsed.sitemapindex.sitemap)
+      .map((s: any) => s?.loc)
+      .filter((loc: any): loc is string => typeof loc === 'string')
+      .slice(0, MAX_SUB_SITEMAPS);
+  }
+
+  if (parsed?.urlset) {
+    rawPageUrls.push(
+      ...toArray(parsed.urlset.url)
+        .map((u: any) => u?.loc)
+        .filter((loc: any): loc is string => typeof loc === 'string'),
+    );
+  }
+
+  // Same-site only (www/protocol-tolerant) — a sitemap listing external URLs shouldn't seed this project's queue.
+  const sameOrigin = rawPageUrls.filter((u) => sameSite(u, origin));
+
+  return { pageUrls: [...new Set(sameOrigin)].slice(0, maxUrls), subSitemapUrls };
+}
+
+async function fetchAndParseXml(url: string, dispatcher?: any): Promise<any | null> {
+  // Same SSRF posture as the crawl itself (assertCrawlUrlSafe, not the
+  // stricter webhook-oriented assertResolvesToPublicAddress) — a sitemap
+  // living on a tenant's internal/VPN-only staging server is a legitimate
+  // crawl target, not an attack; only loopback/cloud-metadata are blocked.
+  try {
+    assertCrawlUrlSafe(url);
+  } catch {
+    return null;
+  }
+  try {
+    // Redirects followed normally here (unlike the webhook-dispatch.ts
+    // 'manual' pattern) — an http→https or bare→www redirect on the
+    // sitemap is completely ordinary, and Playwright's actual page
+    // navigation later in the crawl already follows redirects to whatever
+    // this same site sends it to; a sitemap fetch isn't a more sensitive
+    // operation than that.
+    const res = await fetch(url, {
+      signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
+      headers: { 'User-Agent': 'ZeTa-AI-Crawler/1.0 (+sitemap-discovery)' },
+      ...(dispatcher ? { dispatcher } : {}),
+    });
+    if (!res.ok) return null;
+    const text = await res.text();
+    return new XMLParser().parse(text);
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Fetches and parses a sitemap (or sitemap index, recursed up to a small
+ * depth) into a flat, deduped list of same-origin URLs.
+ */
+export async function fetchSitemapUrls(sitemapUrl: string, origin: string, depth = 0, dispatcher?: any, maxUrls?: number): Promise<string[]> {
+  if (depth > MAX_SITEMAP_INDEX_DEPTH) return [];
+  if (maxUrls === undefined) maxUrls = await getSitemapMaxUrls();
+  const parsed = await fetchAndParseXml(sitemapUrl, dispatcher);
+  if (!parsed) return [];
+
+  const { pageUrls, subSitemapUrls } = extractUrlsFromParsedSitemap(parsed, origin, maxUrls);
+  const collected = [...pageUrls];
+
+  if (subSitemapUrls.length > 0 && collected.length < maxUrls) {
+    const subResults = await Promise.all(
+      subSitemapUrls.slice(0, 10).map(subUrl => fetchSitemapUrls(subUrl, origin, depth + 1, dispatcher, maxUrls))
+    );
+    for (const sub of subResults) collected.push(...sub);
+  }
+
+  return [...new Set(collected)].slice(0, maxUrls);
+}
+
+/**
+ * Discovers a project's sitemap (default `${origin}/sitemap.xml`, or an
+ * explicit override) and returns the URLs it lists. Never throws.
+ */
+export async function discoverSitemapUrls(appUrl: string, sitemapUrl?: string): Promise<string[]> {
+  let origin: string;
+  try {
+    origin = new URL(appUrl).origin;
+  } catch {
+    return [];
+  }
+  const target = sitemapUrl || `${origin}/sitemap.xml`;
+  try {
+    return await fetchSitemapUrls(target, origin);
+  } catch (err) {
+    logger.warn({ err: String(err), target }, '[sitemap] discovery failed, continuing without it');
+    return [];
+  }
+}
+
+/**
+ * Build an undici ProxyAgent from a proxy URL. Parses credentials manually because
+ * DataImpulse-style usernames contain commas (country codes) that break new URL().
+ * Returns undefined for no/auto proxy — the caller then fetches directly.
+ */
+export function buildProxyDispatcher(proxyUrl?: string): ProxyAgent | undefined {
+  if (!proxyUrl || proxyUrl === 'auto') return undefined;
+  try {
+    const m = proxyUrl.match(/^(https?):\/\/(?:([^:@]*)(?::([^@]*))?@)?(.+)$/);
+    if (!m) return new ProxyAgent(proxyUrl);
+    const [, scheme, user, pass, hostPort] = m;
+    const uri = `${scheme}://${hostPort}`;
+    if (user) {
+      const token = 'Basic ' + Buffer.from(`${decodeURIComponent(user)}:${decodeURIComponent(pass ?? '')}`).toString('base64');
+      return new ProxyAgent({ uri, token });
+    }
+    return new ProxyAgent({ uri });
+  } catch {
+    return undefined;
+  }
+}
+
+/** Extract same-origin `Sitemap:` directives from a robots.txt body. */
+export function parseRobotsSitemaps(robotsTxt: string, origin: string): string[] {
+  const out: string[] = [];
+  for (const line of robotsTxt.split(/\r?\n/)) {
+    const m = line.match(/^\s*sitemap:\s*(\S+)/i);
+    if (m && sameSite(m[1], origin)) out.push(m[1]);
+  }
+  return [...new Set(out)];
+}
+
+export interface UrlCandidate {
+  url: string;
+  source: 'sitemap' | 'robots' | 'link';
+}
+
+/**
+ * Pre-crawl URL discovery for the review-before-crawl flow: reads robots.txt Sitemap
+ * directives and sitemap.xml (recursed, same-origin), proxy-aware. Returns a deduped
+ * candidate list the user can review and seed before launching the Playwright crawl.
+ * Never throws — bot-protected sites (Akamai/Cloudflare) simply yield an empty list.
+ */
+export async function discoverProjectUrls(
+  appUrl: string,
+  opts?: { sitemapUrl?: string; proxyUrl?: string },
+): Promise<{ candidates: UrlCandidate[]; sitemapFound: boolean }> {
+  let origin: string;
+  try { origin = new URL(appUrl).origin; } catch { return { candidates: [], sitemapFound: false }; }
+
+  const dispatcher = buildProxyDispatcher(opts?.proxyUrl);
+  const maxUrls = await getSitemapMaxUrls();
+
+  // 1. robots.txt → declared sitemaps
+  let robotsSitemaps: string[] = [];
+  try {
+    assertCrawlUrlSafe(`${origin}/robots.txt`);
+    const res = await fetch(`${origin}/robots.txt`, {
+      signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
+      headers: { 'User-Agent': 'ZeTa-AI-Crawler/1.0 (+url-discovery)' },
+      ...(dispatcher ? { dispatcher } : {}),
+    });
+    if (res.ok) robotsSitemaps = parseRobotsSitemaps(await res.text(), origin);
+  } catch { /* best-effort */ }
+
+  // 2. sitemap targets: explicit override + robots-declared + conventional default
+  const robotsSet = new Set(robotsSitemaps);
+  const sitemapTargets = [...new Set([
+    ...(opts?.sitemapUrl ? [opts.sitemapUrl] : []),
+    ...robotsSitemaps,
+    `${origin}/sitemap.xml`,
+  ])];
+
+  const seen = new Map<string, UrlCandidate>();
+  let sitemapFound = false;
+  for (const sm of sitemapTargets) {
+    const urls = await fetchSitemapUrls(sm, origin, 0, dispatcher, maxUrls);
+    if (urls.length) sitemapFound = true;
+    const source: UrlCandidate['source'] = robotsSet.has(sm) ? 'robots' : 'sitemap';
+    for (const u of urls) if (!seen.has(u)) seen.set(u, { url: u, source });
+  }
+
+  return { candidates: [...seen.values()].slice(0, maxUrls), sitemapFound };
+}

package/src/skyvern.ts

@@ -0,0 +1,199 @@
+/**
+ * Skyvern Vision Fallback
+ *
+ * Skyvern combines computer vision + LLM reasoning to navigate sites that
+ * defeat DOM-based approaches: CAPTCHAs, legacy apps, visual-only login flows,
+ * government portals. It's the "break glass" option when Stagehand fails.
+ *
+ * Production setup (Phase 4+):
+ *   1. Self-host: https://github.com/Skyvern-AI/skyvern
+ *   2. Or use cloud: https://app.skyvern.com
+ *   3. Set SKYVERN_API_URL + SKYVERN_API_KEY in .env
+ *
+ * Pricing note: vision tasks cost more per step than DOM-based Stagehand.
+ * Use only when the fast path fails.
+ */
+
+export interface SkyvernTask {
+  taskDescription: string;
+  url: string;
+  credentials?: { username: string; password: string; oneTimeCode?: string };
+}
+
+export interface SkyvernConfig {
+  apiUrl: string;
+  apiKey: string;
+  llmKey?: string;
+  llmApiKey?: string;
+}
+
+export interface SkyvernStep {
+  order: number;
+  action_type: string;
+  element_description?: string;
+  reasoning?: string;
+  success: boolean;
+  screenshot_url?: string;
+  duration_ms?: number;
+  error?: string;
+}
+
+export interface SkyvernNavigateResult {
+  screenshotUrls: string[];
+  visitedUrls: string[];
+  status: 'success' | 'failed' | 'partial';
+  steps?: SkyvernStep[];
+  taskId?: string;
+  durationMs?: number;
+  triggerReason?: string;
+}
+
+/**
+ * Gap 5: Post-login navigation via Skyvern vision for apps where DOM analysis
+ * cannot discover screens (canvas, WebGL, legacy Flash-replacement stacks).
+ * Skyvern drives the browser via screenshot + LLM, returning visited page list.
+ */
+export async function skyvernNavigate(
+  startUrl: string,
+  navigationGoals: string[],
+  tenantConfig?: SkyvernConfig,
+  triggerReason?: string,
+): Promise<SkyvernNavigateResult> {
+  const apiUrl = tenantConfig?.apiUrl || process.env.SKYVERN_API_URL;
+  const apiKey = tenantConfig?.apiKey || process.env.SKYVERN_API_KEY;
+
+  if (!apiUrl || !apiKey) {
+    console.warn('[skyvern] skyvernNavigate: no API config — skipping post-login vision navigation');
+    return { screenshotUrls: [], visitedUrls: [], status: 'failed' };
+  }
+
+  const visitedUrls: string[] = [];
+  const screenshotUrls: string[] = [];
+  const skyvernSteps: SkyvernStep[] = [];
+  let lastTaskId: string | undefined;
+  let totalDurationMs = 0;
+
+  for (const goal of navigationGoals) {
+    try {
+      const taskStart = Date.now();
+      const res = await fetch(`${apiUrl}/api/v1/tasks`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json', 'x-api-key': apiKey },
+        body: JSON.stringify({
+          url: startUrl,
+          navigation_goal: goal,
+          ...(tenantConfig?.llmKey ? { llm_key: tenantConfig.llmKey } : process.env.SKYVERN_LLM_KEY ? { llm_key: process.env.SKYVERN_LLM_KEY } : {}),
+          ...(tenantConfig?.llmApiKey ? { llm_api_key: tenantConfig.llmApiKey } : process.env.OPENAI_API_KEY ? { llm_api_key: process.env.OPENAI_API_KEY } : {}),
+        }),
+      });
+      if (!res.ok) continue;
+      const data: any = await res.json();
+      const taskId: string = data.task_id;
+      if (!taskId) continue;
+      lastTaskId = taskId;
+
+      // Poll for task completion (max 60s per goal)
+      let done = false;
+      for (let p = 0; p < 12 && !done; p++) {
+        await new Promise((r) => setTimeout(r, 5000));
+        const poll = await fetch(`${apiUrl}/api/v1/tasks/${taskId}`, {
+          headers: { 'x-api-key': apiKey },
+        }).catch(() => null);
+        if (!poll?.ok) continue;
+        const pollData: any = await poll.json();
+        if (pollData.status === 'completed' || pollData.status === 'failed') {
+          done = true;
+          totalDurationMs += Date.now() - taskStart;
+          if (pollData.status === 'completed') {
+            if (pollData.output?.url) visitedUrls.push(pollData.output.url);
+            if (pollData.output?.screenshot_url) screenshotUrls.push(pollData.output.screenshot_url);
+          }
+
+          // Fetch step-level detail from Skyvern
+          try {
+            const stepsRes = await fetch(`${apiUrl}/api/v1/tasks/${taskId}/steps`, {
+              headers: { 'x-api-key': apiKey },
+            }).catch(() => null);
+            if (stepsRes?.ok) {
+              const stepsData: any = await stepsRes.json();
+              const stepsList = Array.isArray(stepsData) ? stepsData : (stepsData?.steps ?? []);
+              for (const s of stepsList) {
+                if (s.output?.screenshot_url) screenshotUrls.push(s.output.screenshot_url);
+                skyvernSteps.push({
+                  order: s.step_order ?? s.order ?? 0,
+                  action_type: s.action_type ?? s.action ?? 'unknown',
+                  element_description: s.output?.element_description,
+                  reasoning: s.output?.llm_response?.action_description ?? s.output?.reasoning,
+                  success: s.status === 'completed',
+                  screenshot_url: s.output?.screenshot_url,
+                  error: s.status === 'failed' ? (s.output?.errors?.[0] ?? 'Step failed') : undefined,
+                });
+              }
+            }
+          } catch { /* non-fatal */ }
+        }
+      }
+    } catch (err) {
+      console.warn(`[skyvern] Navigation goal failed: ${goal}`, err);
+    }
+  }
+
+  return {
+    screenshotUrls,
+    visitedUrls,
+    status: visitedUrls.length > 0 ? 'success' : screenshotUrls.length > 0 ? 'partial' : 'failed',
+    steps: skyvernSteps,
+    taskId: lastTaskId,
+    durationMs: totalDurationMs || undefined,
+    triggerReason,
+  };
+}
+
+export async function skyvernFallback(task: SkyvernTask, tenantConfig?: SkyvernConfig): Promise<'success' | 'failed'> {
+  const apiUrl = tenantConfig?.apiUrl || process.env.SKYVERN_API_URL;
+  const apiKey = tenantConfig?.apiKey || process.env.SKYVERN_API_KEY;
+
+  if (!apiUrl || !apiKey) {
+    console.warn(
+      '[skyvern] Vision fallback disabled — set SKYVERN_API_URL + SKYVERN_API_KEY in .env to enable.',
+      '\n[skyvern] Self-host: https://github.com/Skyvern-AI/skyvern'
+    );
+    return 'failed';
+  }
+
+  try {
+    const res = await fetch(`${apiUrl}/api/v1/tasks`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json', 'x-api-key': apiKey },
+      body: JSON.stringify({
+        url: task.url,
+        navigation_goal: task.taskDescription,
+        ...(task.credentials && {
+          navigation_payload: {
+            username: task.credentials.username,
+            password: task.credentials.password,
+            ...(task.credentials.oneTimeCode && { otp: task.credentials.oneTimeCode }),
+          },
+        }),
+        ...(tenantConfig?.llmKey ? { llm_key: tenantConfig.llmKey } : process.env.SKYVERN_LLM_KEY ? { llm_key: process.env.SKYVERN_LLM_KEY } : {}),
+        ...(tenantConfig?.llmApiKey ? { llm_api_key: tenantConfig.llmApiKey } : process.env.OPENAI_API_KEY ? { llm_api_key: process.env.OPENAI_API_KEY } : {}),
+      }),
+    });
+
+    if (!res.ok) {
+      const body = await res.text().catch(() => '');
+      console.error(`[skyvern] Task creation failed: ${res.status}`, body);
+      return 'failed';
+    }
+
+    const data: any = await res.json();
+    console.log(`[skyvern] Task queued: ${data.task_id}. Poll GET /api/v1/tasks/${data.task_id} for completion.`);
+    // Full implementation: poll until status === "completed" | "failed".
+    // For Phase 1 the login already happened in the browser context Skyvern manages,
+    // so returning 'success' here means we can proceed in that browser session.
+    return 'success';
+  } catch (err) {
+    console.error('[skyvern] Error:', err);
+    return 'failed';
+  }
+}

package/src/sqs-queue.ts

@@ -0,0 +1,129 @@
+import {
+  SQSClient,
+  SendMessageCommand,
+  ReceiveMessageCommand,
+  DeleteMessageCommand,
+  GetQueueAttributesCommand,
+} from '@aws-sdk/client-sqs';
+
+const sqs = new SQSClient({ region: process.env.AWS_REGION ?? 'us-east-1' });
+
+function getQueueUrl(): string {
+  const url = process.env.SQS_QUEUE_URL ?? process.env.SQS_CRAWL_QUEUE_URL;
+  if (!url) throw new Error('SQS_QUEUE_URL not configured');
+  return url;
+}
+
+function getDlqUrl(): string | null {
+  return process.env.SQS_DLQ_URL ?? null;
+}
+
+export async function sendJobToSqs(jobId: string): Promise<void> {
+  await sqs.send(new SendMessageCommand({
+    QueueUrl: getQueueUrl(),
+    MessageBody: JSON.stringify({ jobId }),
+  }));
+}
+
+/** Returns the approximate number of messages in the SQS DLQ. */
+export async function getDlqStats(): Promise<{ approximateMessageCount: number; dlqUrl: string | null }> {
+  const dlqUrl = getDlqUrl();
+  if (!dlqUrl) return { approximateMessageCount: 0, dlqUrl: null };
+  try {
+    const { Attributes = {} } = await sqs.send(new GetQueueAttributesCommand({
+      QueueUrl: dlqUrl,
+      AttributeNames: ['ApproximateNumberOfMessages'],
+    }));
+    return {
+      approximateMessageCount: parseInt(Attributes.ApproximateNumberOfMessages ?? '0', 10),
+      dlqUrl,
+    };
+  } catch (err: any) {
+    console.error('[sqs-dlq] getDlqStats error:', err.message);
+    return { approximateMessageCount: 0, dlqUrl };
+  }
+}
+
+/**
+ * Redrive up to `maxMessages` messages from the SQS DLQ back to the main queue.
+ * Each message is received from the DLQ, re-sent to the main queue, then deleted
+ * from the DLQ. Returns the number of messages successfully redriven.
+ */
+export async function redriveFromDlq(maxMessages = 10): Promise<number> {
+  const dlqUrl = getDlqUrl();
+  if (!dlqUrl) {
+    console.warn('[sqs-dlq] SQS_DLQ_URL not configured — cannot redrive');
+    return 0;
+  }
+  const mainUrl = getQueueUrl();
+  let totalRedriven = 0;
+
+  while (totalRedriven < maxMessages) {
+    const batch = Math.min(10, maxMessages - totalRedriven);
+    const { Messages = [] } = await sqs.send(new ReceiveMessageCommand({
+      QueueUrl: dlqUrl,
+      MaxNumberOfMessages: batch,
+      WaitTimeSeconds: 1,
+    }));
+    if (Messages.length === 0) break;
+
+    for (const msg of Messages) {
+      try {
+        await sqs.send(new SendMessageCommand({
+          QueueUrl: mainUrl,
+          MessageBody: msg.Body!,
+        }));
+        await sqs.send(new DeleteMessageCommand({
+          QueueUrl: dlqUrl,
+          ReceiptHandle: msg.ReceiptHandle!,
+        }));
+        totalRedriven++;
+        console.log('[sqs-dlq] redriven message:', msg.MessageId);
+      } catch (err: any) {
+        console.error('[sqs-dlq] redrive error for message', msg.MessageId, ':', err.message);
+      }
+    }
+  }
+
+  return totalRedriven;
+}
+
+export async function startSqsWorker(): Promise<void> {
+  const queueUrl = getQueueUrl();
+  const { claimSpecificJob, dispatchClaimedJob } = await import('./job-queue.js');
+  console.log('[sqs-worker] starting, queue:', queueUrl);
+
+  while (true) {
+    try {
+      const { Messages = [] } = await sqs.send(new ReceiveMessageCommand({
+        QueueUrl: queueUrl,
+        WaitTimeSeconds: 20,
+        MaxNumberOfMessages: 2,
+      }));
+
+      await Promise.all(Messages.map(async (msg) => {
+        let jobId = '?';
+        try {
+          const data = JSON.parse(msg.Body!);
+          jobId = data.jobId;
+          // Atomically move QUEUED → RUNNING; returns null if another worker beat us.
+          const claimed = await claimSpecificJob(jobId);
+          if (claimed) {
+            await dispatchClaimedJob(claimed);
+          }
+          // Delete whether we ran it or another worker did — message is handled either way.
+          await sqs.send(new DeleteMessageCommand({
+            QueueUrl: queueUrl,
+            ReceiptHandle: msg.ReceiptHandle!,
+          }));
+        } catch (err: any) {
+          console.error(`[sqs-worker] job ${jobId} failed — leaving for SQS redrive:`, err.message);
+          // Do NOT delete — SQS visibility timeout expires and DLQ handles retry.
+        }
+      }));
+    } catch (err: any) {
+      console.error('[sqs-worker] receive error:', err.message);
+      await new Promise((r) => setTimeout(r, 2000));
+    }
+  }
+}

package/src/sso-token.ts

@@ -0,0 +1,120 @@
+/**
+ * SSO token retrieval for Okta and Azure AD.
+ * Uses OAuth2 ROPC (Resource Owner Password Credentials) flow to get access tokens
+ * without browser-based login.
+ *
+ * Note: ROPC is deprecated in OAuth 2.1 but still supported by Okta and Azure AD
+ * for service accounts and automated testing scenarios.
+ */
+
+export type SsoProvider = 'okta' | 'azure_ad' | 'generic_oidc';
+
+export interface SsoConfig {
+  provider: SsoProvider;
+  /** Okta domain (e.g. 'mycompany.okta.com') or Azure tenant ID */
+  domain: string;
+  /** OAuth2 client ID */
+  clientId: string;
+  /** OAuth2 client secret (for client_credentials or confidential ROPC) */
+  clientSecret?: string;
+  /** Scopes to request (space-separated). Default: 'openid profile' */
+  scope?: string;
+}
+
+/**
+ * Get an access token using ROPC flow (username + password → token).
+ * Returns the access token string.
+ */
+export async function getSsoToken(
+  config: SsoConfig,
+  username: string,
+  password: string
+): Promise<string> {
+  let tokenUrl: string;
+  const params: Record<string, string> = {
+    grant_type: 'password',
+    username,
+    password,
+    scope: config.scope ?? 'openid profile',
+    client_id: config.clientId,
+  };
+
+  if (config.clientSecret) {
+    params.client_secret = config.clientSecret;
+  }
+
+  switch (config.provider) {
+    case 'okta':
+      tokenUrl = `https://${config.domain}/oauth2/v1/token`;
+      break;
+    case 'azure_ad':
+      // domain here is the tenant ID (e.g. XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX or 'mycompany.onmicrosoft.com')
+      tokenUrl = `https://login.microsoftonline.com/${config.domain}/oauth2/v2.0/token`;
+      break;
+    case 'generic_oidc':
+      tokenUrl = config.domain.startsWith('http') ? config.domain : `https://${config.domain}/oauth2/token`;
+      break;
+    default:
+      throw new Error(`Unknown SSO provider: ${config.provider}`);
+  }
+
+  const res = await fetch(tokenUrl, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
+    body: new URLSearchParams(params).toString(),
+  });
+
+  if (!res.ok) {
+    const body = await res.text().catch(() => res.statusText);
+    throw new Error(`SSO token request failed (${res.status}): ${body}`);
+  }
+
+  const data = await res.json() as { access_token?: string; error?: string; error_description?: string };
+  if (data.error) throw new Error(`SSO error: ${data.error} — ${data.error_description}`);
+  if (!data.access_token) throw new Error('SSO response missing access_token');
+
+  return data.access_token;
+}
+
+/**
+ * Get a client credentials token (no user — for service accounts).
+ */
+export async function getSsoClientCredentialsToken(
+  config: SsoConfig
+): Promise<string> {
+  if (!config.clientSecret) throw new Error('clientSecret required for client credentials flow');
+
+  let tokenUrl: string;
+  switch (config.provider) {
+    case 'okta':
+      tokenUrl = `https://${config.domain}/oauth2/v1/token`;
+      break;
+    case 'azure_ad':
+      tokenUrl = `https://login.microsoftonline.com/${config.domain}/oauth2/v2.0/token`;
+      break;
+    default:
+      tokenUrl = `https://${config.domain}/oauth2/token`;
+  }
+
+  const res = await fetch(tokenUrl, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
+    body: new URLSearchParams({
+      grant_type: 'client_credentials',
+      client_id: config.clientId,
+      client_secret: config.clientSecret,
+      scope: config.scope ?? 'openid profile',
+    }).toString(),
+  });
+
+  if (!res.ok) {
+    const body = await res.text().catch(() => res.statusText);
+    throw new Error(`SSO client credentials failed (${res.status}): ${body}`);
+  }
+
+  const data = await res.json() as { access_token?: string; error?: string; error_description?: string };
+  if (data.error) throw new Error(`SSO error: ${data.error} — ${data.error_description}`);
+  if (!data.access_token) throw new Error('SSO response missing access_token');
+
+  return data.access_token;
+}