@detiq/crawler 0.1.3

package/src/skyvern.ts

@@ -0,0 +1,199 @@
+/**
+ * Skyvern Vision Fallback
+ *
+ * Skyvern combines computer vision + LLM reasoning to navigate sites that
+ * defeat DOM-based approaches: CAPTCHAs, legacy apps, visual-only login flows,
+ * government portals. It's the "break glass" option when Stagehand fails.
+ *
+ * Production setup (Phase 4+):
+ *   1. Self-host: https://github.com/Skyvern-AI/skyvern
+ *   2. Or use cloud: https://app.skyvern.com
+ *   3. Set SKYVERN_API_URL + SKYVERN_API_KEY in .env
+ *
+ * Pricing note: vision tasks cost more per step than DOM-based Stagehand.
+ * Use only when the fast path fails.
+ */
+
+export interface SkyvernTask {
+  taskDescription: string;
+  url: string;
+  credentials?: { username: string; password: string; oneTimeCode?: string };
+}
+
+export interface SkyvernConfig {
+  apiUrl: string;
+  apiKey: string;
+  llmKey?: string;
+  llmApiKey?: string;
+}
+
+export interface SkyvernStep {
+  order: number;
+  action_type: string;
+  element_description?: string;
+  reasoning?: string;
+  success: boolean;
+  screenshot_url?: string;
+  duration_ms?: number;
+  error?: string;
+}
+
+export interface SkyvernNavigateResult {
+  screenshotUrls: string[];
+  visitedUrls: string[];
+  status: 'success' | 'failed' | 'partial';
+  steps?: SkyvernStep[];
+  taskId?: string;
+  durationMs?: number;
+  triggerReason?: string;
+}
+
+/**
+ * Gap 5: Post-login navigation via Skyvern vision for apps where DOM analysis
+ * cannot discover screens (canvas, WebGL, legacy Flash-replacement stacks).
+ * Skyvern drives the browser via screenshot + LLM, returning visited page list.
+ */
+export async function skyvernNavigate(
+  startUrl: string,
+  navigationGoals: string[],
+  tenantConfig?: SkyvernConfig,
+  triggerReason?: string,
+): Promise<SkyvernNavigateResult> {
+  const apiUrl = tenantConfig?.apiUrl || process.env.SKYVERN_API_URL;
+  const apiKey = tenantConfig?.apiKey || process.env.SKYVERN_API_KEY;
+
+  if (!apiUrl || !apiKey) {
+    console.warn('[skyvern] skyvernNavigate: no API config — skipping post-login vision navigation');
+    return { screenshotUrls: [], visitedUrls: [], status: 'failed' };
+  }
+
+  const visitedUrls: string[] = [];
+  const screenshotUrls: string[] = [];
+  const skyvernSteps: SkyvernStep[] = [];
+  let lastTaskId: string | undefined;
+  let totalDurationMs = 0;
+
+  for (const goal of navigationGoals) {
+    try {
+      const taskStart = Date.now();
+      const res = await fetch(`${apiUrl}/api/v1/tasks`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json', 'x-api-key': apiKey },
+        body: JSON.stringify({
+          url: startUrl,
+          navigation_goal: goal,
+          ...(tenantConfig?.llmKey ? { llm_key: tenantConfig.llmKey } : process.env.SKYVERN_LLM_KEY ? { llm_key: process.env.SKYVERN_LLM_KEY } : {}),
+          ...(tenantConfig?.llmApiKey ? { llm_api_key: tenantConfig.llmApiKey } : process.env.OPENAI_API_KEY ? { llm_api_key: process.env.OPENAI_API_KEY } : {}),
+        }),
+      });
+      if (!res.ok) continue;
+      const data: any = await res.json();
+      const taskId: string = data.task_id;
+      if (!taskId) continue;
+      lastTaskId = taskId;
+
+      // Poll for task completion (max 60s per goal)
+      let done = false;
+      for (let p = 0; p < 12 && !done; p++) {
+        await new Promise((r) => setTimeout(r, 5000));
+        const poll = await fetch(`${apiUrl}/api/v1/tasks/${taskId}`, {
+          headers: { 'x-api-key': apiKey },
+        }).catch(() => null);
+        if (!poll?.ok) continue;
+        const pollData: any = await poll.json();
+        if (pollData.status === 'completed' || pollData.status === 'failed') {
+          done = true;
+          totalDurationMs += Date.now() - taskStart;
+          if (pollData.status === 'completed') {
+            if (pollData.output?.url) visitedUrls.push(pollData.output.url);
+            if (pollData.output?.screenshot_url) screenshotUrls.push(pollData.output.screenshot_url);
+          }
+
+          // Fetch step-level detail from Skyvern
+          try {
+            const stepsRes = await fetch(`${apiUrl}/api/v1/tasks/${taskId}/steps`, {
+              headers: { 'x-api-key': apiKey },
+            }).catch(() => null);
+            if (stepsRes?.ok) {
+              const stepsData: any = await stepsRes.json();
+              const stepsList = Array.isArray(stepsData) ? stepsData : (stepsData?.steps ?? []);
+              for (const s of stepsList) {
+                if (s.output?.screenshot_url) screenshotUrls.push(s.output.screenshot_url);
+                skyvernSteps.push({
+                  order: s.step_order ?? s.order ?? 0,
+                  action_type: s.action_type ?? s.action ?? 'unknown',
+                  element_description: s.output?.element_description,
+                  reasoning: s.output?.llm_response?.action_description ?? s.output?.reasoning,
+                  success: s.status === 'completed',
+                  screenshot_url: s.output?.screenshot_url,
+                  error: s.status === 'failed' ? (s.output?.errors?.[0] ?? 'Step failed') : undefined,
+                });
+              }
+            }
+          } catch { /* non-fatal */ }
+        }
+      }
+    } catch (err) {
+      console.warn(`[skyvern] Navigation goal failed: ${goal}`, err);
+    }
+  }
+
+  return {
+    screenshotUrls,
+    visitedUrls,
+    status: visitedUrls.length > 0 ? 'success' : screenshotUrls.length > 0 ? 'partial' : 'failed',
+    steps: skyvernSteps,
+    taskId: lastTaskId,
+    durationMs: totalDurationMs || undefined,
+    triggerReason,
+  };
+}
+
+export async function skyvernFallback(task: SkyvernTask, tenantConfig?: SkyvernConfig): Promise<'success' | 'failed'> {
+  const apiUrl = tenantConfig?.apiUrl || process.env.SKYVERN_API_URL;
+  const apiKey = tenantConfig?.apiKey || process.env.SKYVERN_API_KEY;
+
+  if (!apiUrl || !apiKey) {
+    console.warn(
+      '[skyvern] Vision fallback disabled — set SKYVERN_API_URL + SKYVERN_API_KEY in .env to enable.',
+      '\n[skyvern] Self-host: https://github.com/Skyvern-AI/skyvern'
+    );
+    return 'failed';
+  }
+
+  try {
+    const res = await fetch(`${apiUrl}/api/v1/tasks`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json', 'x-api-key': apiKey },
+      body: JSON.stringify({
+        url: task.url,
+        navigation_goal: task.taskDescription,
+        ...(task.credentials && {
+          navigation_payload: {
+            username: task.credentials.username,
+            password: task.credentials.password,
+            ...(task.credentials.oneTimeCode && { otp: task.credentials.oneTimeCode }),
+          },
+        }),
+        ...(tenantConfig?.llmKey ? { llm_key: tenantConfig.llmKey } : process.env.SKYVERN_LLM_KEY ? { llm_key: process.env.SKYVERN_LLM_KEY } : {}),
+        ...(tenantConfig?.llmApiKey ? { llm_api_key: tenantConfig.llmApiKey } : process.env.OPENAI_API_KEY ? { llm_api_key: process.env.OPENAI_API_KEY } : {}),
+      }),
+    });
+
+    if (!res.ok) {
+      const body = await res.text().catch(() => '');
+      console.error(`[skyvern] Task creation failed: ${res.status}`, body);
+      return 'failed';
+    }
+
+    const data: any = await res.json();
+    console.log(`[skyvern] Task queued: ${data.task_id}. Poll GET /api/v1/tasks/${data.task_id} for completion.`);
+    // Full implementation: poll until status === "completed" | "failed".
+    // For Phase 1 the login already happened in the browser context Skyvern manages,
+    // so returning 'success' here means we can proceed in that browser session.
+    return 'success';
+  } catch (err) {
+    console.error('[skyvern] Error:', err);
+    return 'failed';
+  }
+}

package/src/sqs-queue.ts

@@ -0,0 +1,49 @@
+import { SQSClient, SendMessageCommand, ReceiveMessageCommand, DeleteMessageCommand } from '@aws-sdk/client-sqs';
+
+const sqs = new SQSClient({ region: process.env.AWS_REGION ?? 'us-east-1' });
+
+export async function sendJobToSqs(jobId: string, tenantId: string, config: any): Promise<void> {
+  const queueUrl = process.env.SQS_CRAWL_QUEUE_URL;
+  if (!queueUrl) throw new Error('SQS_CRAWL_QUEUE_URL not configured');
+  await sqs.send(new SendMessageCommand({
+    QueueUrl: queueUrl,
+    MessageBody: JSON.stringify({ jobId, tenantId, config }),
+  }));
+}
+
+export async function startSqsWorker(): Promise<void> {
+  const queueUrl = process.env.SQS_CRAWL_QUEUE_URL;
+  if (!queueUrl) throw new Error('SQS_CRAWL_QUEUE_URL not configured');
+
+  const { processJob } = await import('./job-queue.js');
+  console.log('[sqs-worker] starting, queue:', queueUrl);
+
+  while (true) {
+    try {
+      const { Messages = [] } = await sqs.send(new ReceiveMessageCommand({
+        QueueUrl: queueUrl,
+        WaitTimeSeconds: 20,
+        MaxNumberOfMessages: 2,
+      }));
+
+      await Promise.all(Messages.map(async (msg) => {
+        let jobId = '?';
+        try {
+          const data = JSON.parse(msg.Body!);
+          jobId = data.jobId;
+          await processJob(data.jobId, data.config);
+          await sqs.send(new DeleteMessageCommand({
+            QueueUrl: queueUrl,
+            ReceiptHandle: msg.ReceiptHandle!,
+          }));
+        } catch (err: any) {
+          console.error(`[sqs-worker] job ${jobId} failed — leaving for SQS redrive:`, err.message);
+          // Do NOT delete — SQS redrive policy + DLQ handles retry
+        }
+      }));
+    } catch (err: any) {
+      console.error('[sqs-worker] receive error:', err.message);
+      await new Promise((r) => setTimeout(r, 2000));
+    }
+  }
+}

package/src/sso-token.ts

@@ -0,0 +1,120 @@
+/**
+ * SSO token retrieval for Okta and Azure AD.
+ * Uses OAuth2 ROPC (Resource Owner Password Credentials) flow to get access tokens
+ * without browser-based login.
+ *
+ * Note: ROPC is deprecated in OAuth 2.1 but still supported by Okta and Azure AD
+ * for service accounts and automated testing scenarios.
+ */
+
+export type SsoProvider = 'okta' | 'azure_ad' | 'generic_oidc';
+
+export interface SsoConfig {
+  provider: SsoProvider;
+  /** Okta domain (e.g. 'mycompany.okta.com') or Azure tenant ID */
+  domain: string;
+  /** OAuth2 client ID */
+  clientId: string;
+  /** OAuth2 client secret (for client_credentials or confidential ROPC) */
+  clientSecret?: string;
+  /** Scopes to request (space-separated). Default: 'openid profile' */
+  scope?: string;
+}
+
+/**
+ * Get an access token using ROPC flow (username + password → token).
+ * Returns the access token string.
+ */
+export async function getSsoToken(
+  config: SsoConfig,
+  username: string,
+  password: string
+): Promise<string> {
+  let tokenUrl: string;
+  const params: Record<string, string> = {
+    grant_type: 'password',
+    username,
+    password,
+    scope: config.scope ?? 'openid profile',
+    client_id: config.clientId,
+  };
+
+  if (config.clientSecret) {
+    params.client_secret = config.clientSecret;
+  }
+
+  switch (config.provider) {
+    case 'okta':
+      tokenUrl = `https://${config.domain}/oauth2/v1/token`;
+      break;
+    case 'azure_ad':
+      // domain here is the tenant ID (e.g. XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX or 'mycompany.onmicrosoft.com')
+      tokenUrl = `https://login.microsoftonline.com/${config.domain}/oauth2/v2.0/token`;
+      break;
+    case 'generic_oidc':
+      tokenUrl = config.domain.startsWith('http') ? config.domain : `https://${config.domain}/oauth2/token`;
+      break;
+    default:
+      throw new Error(`Unknown SSO provider: ${config.provider}`);
+  }
+
+  const res = await fetch(tokenUrl, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
+    body: new URLSearchParams(params).toString(),
+  });
+
+  if (!res.ok) {
+    const body = await res.text().catch(() => res.statusText);
+    throw new Error(`SSO token request failed (${res.status}): ${body}`);
+  }
+
+  const data = await res.json() as { access_token?: string; error?: string; error_description?: string };
+  if (data.error) throw new Error(`SSO error: ${data.error} — ${data.error_description}`);
+  if (!data.access_token) throw new Error('SSO response missing access_token');
+
+  return data.access_token;
+}
+
+/**
+ * Get a client credentials token (no user — for service accounts).
+ */
+export async function getSsoClientCredentialsToken(
+  config: SsoConfig
+): Promise<string> {
+  if (!config.clientSecret) throw new Error('clientSecret required for client credentials flow');
+
+  let tokenUrl: string;
+  switch (config.provider) {
+    case 'okta':
+      tokenUrl = `https://${config.domain}/oauth2/v1/token`;
+      break;
+    case 'azure_ad':
+      tokenUrl = `https://login.microsoftonline.com/${config.domain}/oauth2/v2.0/token`;
+      break;
+    default:
+      tokenUrl = `https://${config.domain}/oauth2/token`;
+  }
+
+  const res = await fetch(tokenUrl, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
+    body: new URLSearchParams({
+      grant_type: 'client_credentials',
+      client_id: config.clientId,
+      client_secret: config.clientSecret,
+      scope: config.scope ?? 'openid profile',
+    }).toString(),
+  });
+
+  if (!res.ok) {
+    const body = await res.text().catch(() => res.statusText);
+    throw new Error(`SSO client credentials failed (${res.status}): ${body}`);
+  }
+
+  const data = await res.json() as { access_token?: string; error?: string; error_description?: string };
+  if (data.error) throw new Error(`SSO error: ${data.error} — ${data.error_description}`);
+  if (!data.access_token) throw new Error('SSO response missing access_token');
+
+  return data.access_token;
+}

package/src/stealth/human-behavior.ts

@@ -0,0 +1,191 @@
+import type { Page } from 'playwright';
+
+// Gaussian random using Box-Muller transform
+function gaussian(mean: number, sigma: number): number {
+  const u1 = Math.random();
+  const u2 = Math.random();
+  const z = Math.sqrt(-2 * Math.log(u1)) * Math.cos(2 * Math.PI * u2);
+  return mean + z * sigma;
+}
+
+export async function humanDelay(min = 80, max = 450): Promise<void> {
+  const mean = (min + max) / 2;
+  const sigma = (max - min) / 6;
+  const delay = Math.max(min, Math.min(max, gaussian(mean, sigma)));
+  await new Promise(r => setTimeout(r, Math.round(delay)));
+}
+
+export async function simulateMouseMovement(page: Page, targetX: number, targetY: number): Promise<void> {
+  try {
+    const mouse = page.mouse;
+    const steps = 8 + Math.floor(Math.random() * 8);
+    // Generate Bezier control points with slight perpendicular offset
+    const startX = targetX - 100 + Math.random() * 200;
+    const startY = targetY - 100 + Math.random() * 200;
+    const cpX = (startX + targetX) / 2 + (Math.random() - 0.5) * 80;
+    const cpY = (startY + targetY) / 2 + (Math.random() - 0.5) * 80;
+
+    for (let i = 0; i <= steps; i++) {
+      const t = i / steps;
+      // Quadratic Bezier: B(t) = (1-t)^2 * P0 + 2(1-t)t * P1 + t^2 * P2
+      const x = (1 - t) * (1 - t) * startX + 2 * (1 - t) * t * cpX + t * t * targetX;
+      const y = (1 - t) * (1 - t) * startY + 2 * (1 - t) * t * cpY + t * t * targetY;
+      await mouse.move(x + (Math.random() - 0.5) * 2, y + (Math.random() - 0.5) * 2);
+      if (i < steps) await humanDelay(8, 25);
+    }
+    // Small overshoot correction
+    await mouse.move(targetX + (Math.random() - 0.5) * 4, targetY + (Math.random() - 0.5) * 4);
+    await humanDelay(30, 80);
+    await mouse.move(targetX, targetY);
+  } catch { /* non-fatal */ }
+}
+
+export async function simulateMouseHover(page: Page, selector: string): Promise<void> {
+  try {
+    const el = await page.$(selector);
+    if (!el) return;
+    const box = await el.boundingBox();
+    if (!box) return;
+    const cx = box.x + box.width / 2;
+    const cy = box.y + box.height / 2;
+    await simulateMouseMovement(page, cx, cy);
+    await humanDelay(100, 300);
+  } catch { /* non-fatal */ }
+}
+
+export async function simulateClick(page: Page, selector: string): Promise<void> {
+  try {
+    await simulateMouseHover(page, selector);
+    await humanDelay(50, 150);
+    await page.click(selector, { delay: Math.round(gaussian(80, 30)) });
+  } catch { /* non-fatal */ }
+}
+
+export async function injectFingerprintOverrides(page: Page): Promise<void> {
+  await page.addInitScript(() => {
+    // Canvas noise — subtle per-pixel perturbation
+    const origToDataURL = HTMLCanvasElement.prototype.toDataURL;
+    HTMLCanvasElement.prototype.toDataURL = function(this: HTMLCanvasElement, ...args: any[]) {
+      const ctx = this.getContext('2d');
+      if (ctx) {
+        const imageData = ctx.getImageData(0, 0, this.width, this.height);
+        for (let i = 0; i < imageData.data.length; i += 4) {
+          imageData.data[i] = Math.max(0, Math.min(255, imageData.data[i] + (Math.random() < 0.01 ? (Math.random() > 0.5 ? 1 : -1) : 0)));
+        }
+        ctx.putImageData(imageData, 0, 0);
+      }
+      return origToDataURL.apply(this, args as [type?: string, quality?: number]);
+    };
+
+    // WebGL parameter spoofing
+    const origGetParam = WebGLRenderingContext.prototype.getParameter;
+    WebGLRenderingContext.prototype.getParameter = function(this: WebGLRenderingContext, parameter: number) {
+      const UNMASKED_VENDOR_WEBGL = 0x9245;
+      const UNMASKED_RENDERER_WEBGL = 0x9246;
+      if (parameter === UNMASKED_VENDOR_WEBGL) return 'Intel Inc.';
+      if (parameter === UNMASKED_RENDERER_WEBGL) return 'Intel Iris OpenGL Engine';
+      return origGetParam.call(this, parameter);
+    };
+
+    // Hardware concurrency — pick from common values
+    const cpuCounts = [4, 6, 8, 8];
+    Object.defineProperty(navigator, 'hardwareConcurrency', {
+      get: () => cpuCounts[Math.floor(Math.random() * cpuCounts.length)],
+      configurable: true,
+    });
+
+    // Device memory
+    Object.defineProperty(navigator, 'deviceMemory', {
+      get: () => [4, 8][Math.floor(Math.random() * 2)],
+      configurable: true,
+    });
+  }).catch(() => {});
+}
+
+export async function infiniteScrollToBottom(
+  page: Page,
+  opts: { maxScrolls?: number; scrollDelay?: number; stabilizeMs?: number } = {}
+): Promise<{ scrollCount: number; heightReached: number }> {
+  const { maxScrolls = 50, scrollDelay = 400, stabilizeMs = 600 } = opts;
+  let scrollCount = 0;
+  let sameHeightCount = 0;
+  let lastHeight = 0;
+
+  try {
+    while (scrollCount < maxScrolls) {
+      const currentHeight: number = await page.evaluate(() => document.body.scrollHeight).catch(() => 0);
+      if (currentHeight === lastHeight) {
+        sameHeightCount++;
+        if (sameHeightCount >= 3) break; // no new content
+      } else {
+        sameHeightCount = 0;
+        lastHeight = currentHeight;
+      }
+
+      const scrollBy = 500 + Math.floor(Math.random() * 300);
+      await page.evaluate((by: number) => window.scrollBy(0, by), scrollBy).catch(() => {});
+      await humanDelay(scrollDelay - 100, scrollDelay + 200);
+
+      // Wait for potential lazy-loaded content
+      await page.waitForTimeout(stabilizeMs).catch(() => {});
+      scrollCount++;
+    }
+  } catch { /* non-fatal */ }
+
+  return { scrollCount, heightReached: lastHeight };
+}
+
+export async function extractIframeContent(page: Page): Promise<Array<{ src: string; content: string; markdown: string }>> {
+  const results: Array<{ src: string; content: string; markdown: string }> = [];
+  try {
+    const iframeSrcs: string[] = await page.evaluate(() =>
+      Array.from(document.querySelectorAll('iframe')).map(f => f.getAttribute('src') ?? f.getAttribute('data-src') ?? '')
+    ).catch(() => []);
+
+    for (const frame of page.frames()) {
+      const src = frame.url();
+      if (!src || src === 'about:blank' || src === page.url()) continue;
+      try {
+        const content = await frame.content().catch(() => '');
+        if (!content || content.length < 50) continue;
+        // Simple text extraction from iframe HTML
+        const text = content.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '')
+          .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '')
+          .replace(/<[^>]+>/g, ' ')
+          .replace(/\s+/g, ' ')
+          .trim();
+        results.push({ src, content: content.slice(0, 10_000), markdown: text.slice(0, 5_000) });
+      } catch { /* skip this frame */ }
+    }
+  } catch { /* non-fatal */ }
+  return results;
+}
+
+export async function pierceShadowDom(page: Page): Promise<string> {
+  try {
+    return await page.evaluate(() => {
+      const parts: string[] = [];
+      function walkShadow(root: Element | ShadowRoot): void {
+        const shadow = (root as any).shadowRoot;
+        if (shadow) {
+          parts.push(shadow.innerHTML ?? '');
+          for (const child of [...shadow.querySelectorAll('*')]) walkShadow(child);
+        }
+        for (const child of Array.from(root.querySelectorAll(':not(script):not(style)'))) {
+          if ((child as any).shadowRoot) walkShadow(child);
+        }
+      }
+      walkShadow(document.body);
+      return parts
+        .join('\n')
+        .replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '')
+        .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '')
+        .replace(/<[^>]+>/g, ' ')
+        .replace(/\s+/g, ' ')
+        .trim()
+        .slice(0, 10_000);
+    }).catch(() => '');
+  } catch {
+    return '';
+  }
+}

package/src/telemetry.ts

@@ -0,0 +1,122 @@
+/**
+ * Lightweight OTel-compatible tracing for Zeta crawler.
+ *
+ * Emits spans as structured JSON logs compatible with OpenTelemetry format.
+ * Drop-in upgrade path: replace this module with @opentelemetry/sdk-node when needed.
+ *
+ * Usage:
+ *   const span = startSpan('crawl.url', { url, engine });
+ *   // ... do work ...
+ *   endSpan(span, { screensFound: 3 });
+ */
+
+import { randomBytes } from 'node:crypto';
+
+export interface Span {
+  traceId: string;
+  spanId: string;
+  parentSpanId?: string;
+  name: string;
+  startTimeMs: number;
+  attributes: Record<string, string | number | boolean>;
+}
+
+export interface FinishedSpan extends Span {
+  endTimeMs: number;
+  durationMs: number;
+  status: 'ok' | 'error';
+  error?: string;
+}
+
+// Active trace context — per async execution context (simplified: per-process global)
+let currentTraceId: string | null = null;
+let currentSpanId: string | null = null;
+
+function genId(bytes: number): string {
+  return randomBytes(bytes).toString('hex');
+}
+
+/** Start a new span. If a trace is already active, the new span is a child. */
+export function startSpan(name: string, attributes: Record<string, string | number | boolean> = {}): Span {
+  const traceId = currentTraceId ?? genId(16);
+  const spanId = genId(8);
+  const parentSpanId = currentSpanId ?? undefined;
+
+  if (!currentTraceId) currentTraceId = traceId;
+  currentSpanId = spanId;
+
+  return { traceId, spanId, parentSpanId, name, startTimeMs: Date.now(), attributes };
+}
+
+/** End a span and emit it as a structured log line. */
+export function endSpan(span: Span, extraAttributes: Record<string, string | number | boolean> = {}, error?: Error): FinishedSpan {
+  const endTimeMs = Date.now();
+  const durationMs = endTimeMs - span.startTimeMs;
+  const finished: FinishedSpan = {
+    ...span,
+    attributes: { ...span.attributes, ...extraAttributes },
+    endTimeMs,
+    durationMs,
+    status: error ? 'error' : 'ok',
+    error: error?.message,
+  };
+
+  // Reset context when root span ends
+  if (!span.parentSpanId) {
+    currentTraceId = null;
+    currentSpanId = span.parentSpanId ?? null;
+  } else {
+    currentSpanId = span.parentSpanId ?? null;
+  }
+
+  // Emit as structured JSON — compatible with Datadog/Grafana log parsing
+  console.log(JSON.stringify({
+    ts: new Date(endTimeMs).toISOString(),
+    level: error ? 'error' : 'info',
+    msg: `span:${finished.name}`,
+    trace_id: finished.traceId,
+    span_id: finished.spanId,
+    parent_span_id: finished.parentSpanId,
+    duration_ms: durationMs,
+    status: finished.status,
+    error: finished.error,
+    ...finished.attributes,
+  }));
+
+  return finished;
+}
+
+/** Convenience: wrap an async function in a span. */
+export async function withSpan<T>(
+  name: string,
+  attributes: Record<string, string | number | boolean>,
+  fn: (span: Span) => Promise<T>,
+): Promise<T> {
+  const span = startSpan(name, attributes);
+  try {
+    const result = await fn(span);
+    endSpan(span);
+    return result;
+  } catch (err: any) {
+    endSpan(span, {}, err);
+    throw err;
+  }
+}
+
+/** Counter metrics — increment and log on flush. */
+const counters: Map<string, number> = new Map();
+
+export function increment(metric: string, value = 1, tags: Record<string, string> = {}): void {
+  const key = metric + (Object.keys(tags).length ? ':' + Object.entries(tags).map(([k, v]) => `${k}=${v}`).join(',') : '');
+  counters.set(key, (counters.get(key) ?? 0) + value);
+}
+
+/** Flush counters as a structured log — call periodically (e.g. at job end). */
+export function flushMetrics(): Record<string, number> {
+  const snapshot = Object.fromEntries(counters);
+  if (Object.keys(snapshot).length > 0) {
+    console.log(JSON.stringify({ ts: new Date().toISOString(), level: 'info', msg: 'metrics:flush', ...snapshot }));
+    counters.clear();
+  }
+  return snapshot;
+}