@detiq/crawler 0.1.3

package/dist/native-crawler.js

@@ -0,0 +1,839 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+import os from "node:os";
+import { withTenant } from "@qip/database";
+import { AppBrain, getTenantStorageConfig, uploadScreenshotToTenantStorage } from "@qip/app-brain";
+import { config } from "@qip/core";
+import { logger } from "./logger.js";
+function buildCapabilities(opts, localAppPath) {
+  const appiumServerUrl = opts.appiumServerUrl ?? process.env.APPIUM_SERVER_URL ?? "http://127.0.0.1:4723";
+  logger.info({ appiumServerUrl, platform: opts.platform }, "Building Appium capabilities");
+  if (opts.deviceFarm === "browserstack" && (opts.platform === "android" || opts.platform === "ios")) {
+    const bstackOptions = {
+      userName: opts.browserstackUsername ?? process.env.BROWSERSTACK_USERNAME ?? "",
+      accessKey: opts.browserstackAccessKey ?? process.env.BROWSERSTACK_ACCESS_KEY ?? "",
+      deviceName: opts.deviceModel ?? (opts.platform === "android" ? "Google Pixel 7" : "iPhone 15"),
+      osVersion: opts.osVersion ?? (opts.platform === "android" ? "13.0" : "17"),
+      realMobile: true,
+      projectName: "ZeTa-QIP",
+      buildName: `${opts.platform}-crawl`,
+      sessionName: opts.jobId
+    };
+    if (opts.platform === "android") {
+      return {
+        "platformName": "android",
+        "appium:automationName": "UiAutomator2",
+        "appium:app": localAppPath,
+        "appium:newCommandTimeout": 120,
+        "appium:autoGrantPermissions": true,
+        ...opts.appActivity ? { "appium:appActivity": opts.appActivity } : {},
+        ...opts.packageName ? { "appium:appPackage": opts.packageName } : {},
+        "bstack:options": bstackOptions
+      };
+    }
+    return {
+      "platformName": "ios",
+      "appium:automationName": "XCUITest",
+      "appium:app": localAppPath,
+      "appium:newCommandTimeout": 120,
+      ...opts.packageName ? { "appium:bundleId": opts.packageName } : {},
+      "bstack:options": bstackOptions
+    };
+  }
+  if (opts.platform === "android") {
+    return {
+      "appium:platformName": "Android",
+      "appium:deviceName": opts.deviceModel ?? "Pixel 7",
+      "appium:app": localAppPath,
+      "appium:automationName": "UiAutomator2",
+      "appium:autoGrantPermissions": true,
+      "appium:newCommandTimeout": 120,
+      "appium:platformVersion": opts.osVersion ?? "",
+      ...opts.appActivity ? { "appium:appActivity": opts.appActivity } : {},
+      ...opts.appWaitActivity ? { "appium:appWaitActivity": opts.appWaitActivity } : {},
+      "appium:eventTimings": true,
+      // capture launch time telemetry
+      ...opts.packageName ? { "appium:appPackage": opts.packageName } : {}
+    };
+  }
+  if (opts.platform === "windows") {
+    return {
+      "platformName": "Windows",
+      "appium:automationName": "Windows",
+      "appium:app": localAppPath,
+      "appium:deviceName": "WindowsPC",
+      "appium:newCommandTimeout": 120,
+      ...opts.packageName ? { "appium:appTopLevelWindow": opts.packageName } : {}
+    };
+  }
+  if (opts.platform === "macos") {
+    return {
+      "platformName": "mac",
+      "appium:automationName": "Mac2",
+      "appium:app": localAppPath,
+      "appium:deviceName": "Mac",
+      "appium:newCommandTimeout": 120,
+      ...opts.packageName ? { "appium:bundleId": opts.packageName } : {}
+    };
+  }
+  return {
+    "appium:platformName": "iOS",
+    "appium:deviceName": opts.deviceModel ?? "iPhone 15",
+    "appium:app": localAppPath,
+    "appium:automationName": "XCUITest",
+    "appium:newCommandTimeout": 120,
+    "appium:platformVersion": opts.osVersion ?? "",
+    ...opts.packageName ? { "appium:bundleId": opts.packageName } : {}
+  };
+}
+async function uploadToBrowserStack(localPath, opts) {
+  const username = opts.browserstackUsername ?? process.env.BROWSERSTACK_USERNAME ?? "";
+  const accessKey = opts.browserstackAccessKey ?? process.env.BROWSERSTACK_ACCESS_KEY ?? "";
+  const credentials = Buffer.from(`${username}:${accessKey}`).toString("base64");
+  const fileBuffer = await fs.readFile(localPath);
+  const filename = localPath.split("/").pop() ?? "app.apk";
+  const form = new global.FormData();
+  form.append("file", new Blob([new Uint8Array(fileBuffer)]), filename);
+  logger.info({ localPath, filename }, "Uploading app binary to BrowserStack");
+  const response = await fetch("https://api-cloud.browserstack.com/app-automate/upload", {
+    method: "POST",
+    headers: { Authorization: `Basic ${credentials}` },
+    body: form
+  });
+  if (!response.ok) {
+    throw new Error(`BrowserStack upload failed: ${response.status} ${await response.text()}`);
+  }
+  const json = await response.json();
+  if (!json.app_url) throw new Error("BrowserStack upload did not return app_url");
+  logger.info({ appUrl: json.app_url }, "App uploaded to BrowserStack");
+  return json.app_url;
+}
+async function downloadAppFromS3(appS3Path, platform, opts, storageCfg) {
+  if (appS3Path.startsWith("local://")) {
+    const localPath = appS3Path.slice("local://".length);
+    logger.info({ localPath }, "Using locally stored app binary");
+    if (opts.deviceFarm === "browserstack") {
+      return uploadToBrowserStack(localPath, opts);
+    }
+    return localPath;
+  }
+  const withoutScheme = appS3Path.replace(/^s3:\/\//, "");
+  const slashIdx = withoutScheme.indexOf("/");
+  if (slashIdx === -1) {
+    throw new Error(`Invalid S3 path: ${appS3Path}. Expected format: s3://bucket/path/to/file`);
+  }
+  const bucket = withoutScheme.slice(0, slashIdx);
+  const key = withoutScheme.slice(slashIdx + 1);
+  const extMap = { android: ".apk", ios: ".ipa", windows: ".exe", macos: ".app" };
+  const ext = extMap[platform] ?? "";
+  const tmpPath = path.join(os.tmpdir(), `zeta-native-${Date.now()}${ext}`);
+  logger.info({ bucket, key, tmpPath, provider: storageCfg?.provider ?? "env" }, "Downloading app binary from storage");
+  const { S3Client, GetObjectCommand } = await import("@aws-sdk/client-s3");
+  const isR2 = storageCfg?.provider === "R2";
+  const s3ClientCfg = {
+    region: isR2 ? "auto" : storageCfg?.region ?? process.env.AWS_REGION ?? "us-east-1"
+  };
+  if (storageCfg?.accessKeyId && storageCfg?.secretAccessKey) {
+    s3ClientCfg.credentials = { accessKeyId: storageCfg.accessKeyId, secretAccessKey: storageCfg.secretAccessKey };
+  }
+  if (isR2 && storageCfg?.accountId) {
+    s3ClientCfg.endpoint = `https://${storageCfg.accountId}.r2.cloudflarestorage.com`;
+    s3ClientCfg.forcePathStyle = true;
+  }
+  const s3 = new S3Client(s3ClientCfg);
+  const response = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key }));
+  const { Readable } = await import("node:stream");
+  const { pipeline } = await import("node:stream/promises");
+  const fh = await fs.open(tmpPath, "w");
+  try {
+    await pipeline(Readable.from(response.Body), fh.createWriteStream());
+  } finally {
+    await fh.close();
+  }
+  logger.info({ tmpPath }, "App binary downloaded");
+  if (opts.deviceFarm === "browserstack") {
+    return uploadToBrowserStack(tmpPath, opts);
+  }
+  return tmpPath;
+}
+function parseAccessibilityTree(xmlSource, platform) {
+  const elements = [];
+  if (platform === "windows") {
+    const WIN_INTERACTIVE = /* @__PURE__ */ new Set(["Button", "Edit", "ComboBox", "CheckBox", "RadioButton", "Slider", "ListItem", "MenuItem", "Hyperlink"]);
+    const nodeRegex = /<(\w+)\s([^>]*)\/>/g;
+    let match;
+    while ((match = nodeRegex.exec(xmlSource)) !== null) {
+      const typeName = match[1] ?? "";
+      if (!WIN_INTERACTIVE.has(typeName)) continue;
+      const attrs = match[2] ?? "";
+      const get = (name) => new RegExp(`\\b${name}="([^"]*)"`, "i").exec(attrs)?.[1] ?? "";
+      if (get("IsEnabled") === "false" || get("IsOffscreen") === "true") continue;
+      const label = get("Name") || get("AutomationId") || "";
+      if (!label) continue;
+      const role = winTypeToRole(typeName);
+      const brStr = get("BoundingRectangle");
+      const brMatch = /x:(\d+)\s+y:(\d+)\s+width:(\d+)\s+height:(\d+)/i.exec(brStr);
+      const bounds = brMatch ? { x: parseInt(brMatch[1]), y: parseInt(brMatch[2]), width: parseInt(brMatch[3]), height: parseInt(brMatch[4]) } : { x: 0, y: 0, width: 100, height: 30 };
+      elements.push({ role, label, bounds, platform: "windows", className: typeName });
+    }
+  }
+  if (platform === "macos") {
+    const MAC_INTERACTIVE = /* @__PURE__ */ new Set([
+      "XCUIElementTypeButton",
+      "XCUIElementTypeTextField",
+      "XCUIElementTypeSecureTextField",
+      "XCUIElementTypeCheckBox",
+      "XCUIElementTypeRadioButton",
+      "XCUIElementTypePopUpButton",
+      "XCUIElementTypeComboBox",
+      "XCUIElementTypeSlider",
+      "XCUIElementTypeMenuItem",
+      "XCUIElementTypeSearchField",
+      "XCUIElementTypeLink"
+    ]);
+    const nodeRegex = /<(XCUIElementType\w+)\s([^>]*)\/>/g;
+    let match;
+    while ((match = nodeRegex.exec(xmlSource)) !== null) {
+      const typeName = match[1] ?? "";
+      if (!MAC_INTERACTIVE.has(typeName)) continue;
+      const attrs = match[2] ?? "";
+      const get = (name) => new RegExp(`\\b${name}="([^"]*)"`, "i").exec(attrs)?.[1] ?? "";
+      if (get("enabled") === "false") continue;
+      const label = get("title") || get("label") || get("value") || "";
+      if (!label) continue;
+      const x = parseFloat(get("x") || "0");
+      const y = parseFloat(get("y") || "0");
+      const width = parseFloat(get("width") || "0");
+      const height = parseFloat(get("height") || "0");
+      const role = macTypeToRole(typeName);
+      elements.push({ role, label, bounds: { x, y, width, height }, platform: "macos", className: typeName });
+    }
+  }
+  if (platform === "android") {
+    const INTERACTIVE_CLASSES = /* @__PURE__ */ new Set([
+      "android.widget.Button",
+      "android.widget.EditText",
+      "android.widget.CheckBox",
+      "android.widget.RadioButton",
+      "android.widget.Switch",
+      "android.widget.Spinner",
+      "android.widget.TextView",
+      // tappable text
+      "android.widget.ImageButton",
+      "android.widget.ToggleButton",
+      "androidx.recyclerview.widget.RecyclerView",
+      "com.google.android.material.bottomnavigation.BottomNavigationItemView",
+      "android.widget.ImageView"
+      // tappable images (nav icons)
+    ]);
+    const nodeRegex = /<node\s([^>]*)\/?>/g;
+    let match;
+    while ((match = nodeRegex.exec(xmlSource)) !== null) {
+      const attrs = match[1] ?? "";
+      const get = (name) => {
+        const m = new RegExp(`\\b${name}="([^"]*)"`, "i").exec(attrs);
+        return m ? m[1] ?? "" : "";
+      };
+      const className = get("class");
+      const clickable = get("clickable") === "true";
+      const enabled = get("enabled") === "true";
+      if (!enabled) continue;
+      if (!clickable && !INTERACTIVE_CLASSES.has(className)) continue;
+      const label = get("content-desc") || get("text") || get("resource-id").split("/").pop() || "";
+      if (!label) continue;
+      const resourceId = get("resource-id");
+      const boundsStr = get("bounds");
+      const bounds = parseAndroidBounds(boundsStr);
+      if (!bounds) continue;
+      const role = classToRole(className);
+      elements.push({ role, label, bounds, platform: "android", resourceId, className });
+    }
+  } else {
+    const INTERACTIVE_TYPES = /* @__PURE__ */ new Set([
+      "XCUIElementTypeButton",
+      "XCUIElementTypeTextField",
+      "XCUIElementTypeSecureTextField",
+      "XCUIElementTypeSwitch",
+      "XCUIElementTypeSlider",
+      "XCUIElementTypeCheckBox",
+      "XCUIElementTypeRadioButton",
+      "XCUIElementTypePickerWheel",
+      "XCUIElementTypeSearchField",
+      "XCUIElementTypeCell",
+      "XCUIElementTypeLink",
+      "XCUIElementTypeTabBarItem",
+      "XCUIElementTypeNavigationBar"
+    ]);
+    const nodeRegex = /<(XCUIElementType\w+)\s([^>]*)\/>/g;
+    let match;
+    while ((match = nodeRegex.exec(xmlSource)) !== null) {
+      const typeName = match[1] ?? "";
+      if (!INTERACTIVE_TYPES.has(typeName)) continue;
+      const attrs = match[2] ?? "";
+      const get = (name) => {
+        const m = new RegExp(`\\b${name}="([^"]*)"`, "i").exec(attrs);
+        return m ? m[1] ?? "" : "";
+      };
+      if (get("enabled") === "false") continue;
+      const label = get("label") || get("name") || get("value") || "";
+      if (!label) continue;
+      const x = parseFloat(get("x") || "0");
+      const y = parseFloat(get("y") || "0");
+      const width = parseFloat(get("width") || "0");
+      const height = parseFloat(get("height") || "0");
+      if (width === 0 && height === 0) continue;
+      const role = xcuiTypeToRole(typeName);
+      elements.push({ role, label, bounds: { x, y, width, height }, platform: "ios", className: typeName });
+    }
+  }
+  return elements;
+}
+function parseAndroidBounds(boundsStr) {
+  const m = /\[(\d+),(\d+)\]\[(\d+),(\d+)\]/.exec(boundsStr);
+  if (!m) return null;
+  const x1 = parseInt(m[1] ?? "0", 10);
+  const y1 = parseInt(m[2] ?? "0", 10);
+  const x2 = parseInt(m[3] ?? "0", 10);
+  const y2 = parseInt(m[4] ?? "0", 10);
+  return { x: x1, y: y1, width: x2 - x1, height: y2 - y1 };
+}
+function classToRole(className) {
+  if (className.includes("Button")) return "button";
+  if (className.includes("EditText")) return "input";
+  if (className.includes("CheckBox")) return "checkbox";
+  if (className.includes("RadioButton")) return "radio";
+  if (className.includes("Switch") || className.includes("Toggle")) return "switch";
+  if (className.includes("Spinner")) return "select";
+  if (className.includes("TextView")) return "text";
+  return className.split(".").pop()?.toLowerCase() ?? "unknown";
+}
+function xcuiTypeToRole(xcuiType) {
+  if (xcuiType.includes("Button")) return "button";
+  if (xcuiType.includes("TextField")) return "input";
+  if (xcuiType.includes("SecureTextField")) return "input";
+  if (xcuiType.includes("Switch")) return "switch";
+  if (xcuiType.includes("Slider")) return "slider";
+  if (xcuiType.includes("CheckBox")) return "checkbox";
+  if (xcuiType.includes("RadioButton")) return "radio";
+  if (xcuiType.includes("PickerWheel")) return "select";
+  if (xcuiType.includes("SearchField")) return "input";
+  return xcuiType.replace("XCUIElementType", "").toLowerCase();
+}
+function winTypeToRole(winType) {
+  if (winType === "Button") return "button";
+  if (winType === "Edit") return "input";
+  if (winType === "ComboBox") return "select";
+  if (winType === "CheckBox") return "checkbox";
+  if (winType === "RadioButton") return "radio";
+  if (winType === "Slider") return "slider";
+  if (winType === "ListItem") return "option";
+  if (winType === "MenuItem") return "menuitem";
+  if (winType === "Hyperlink") return "link";
+  return winType.toLowerCase();
+}
+function macTypeToRole(xcuiType) {
+  if (xcuiType.includes("Button")) return "button";
+  if (xcuiType.includes("SecureTextField")) return "input";
+  if (xcuiType.includes("TextField")) return "input";
+  if (xcuiType.includes("CheckBox")) return "checkbox";
+  if (xcuiType.includes("RadioButton")) return "radio";
+  if (xcuiType.includes("PopUpButton")) return "select";
+  if (xcuiType.includes("ComboBox")) return "select";
+  if (xcuiType.includes("Slider")) return "slider";
+  if (xcuiType.includes("MenuItem")) return "menuitem";
+  if (xcuiType.includes("SearchField")) return "input";
+  if (xcuiType.includes("Link")) return "link";
+  return xcuiType.replace("XCUIElementType", "").toLowerCase();
+}
+async function attemptNativeLogin(driver, platform, credentials) {
+  try {
+    const pageXml = await driver.getPageSource();
+    const elements = parseAccessibilityTree(pageXml, platform);
+    const inputEls = elements.filter((e) => e.role === "input");
+    if (inputEls.length === 0) return false;
+    const isPasswordField = (e) => e.className?.toLowerCase().includes("secure") || e.className?.toLowerCase().includes("password") || e.label?.toLowerCase().match(/password|passcode|pin|secret/i) != null;
+    const hasPasswordInput = elements.some(isPasswordField);
+    if (!hasPasswordInput && inputEls.length < 2) return false;
+    logger.info({ platform }, "Native auth: login screen detected \u2014 filling credentials");
+    const unLabel = credentials.usernameFieldLabel?.toLowerCase() ?? "";
+    const usernameEl = inputEls.find((e) => {
+      if (isPasswordField(e)) return false;
+      if (unLabel) return e.label?.toLowerCase().includes(unLabel);
+      return e.label?.toLowerCase().match(/email|username|user|login|phone|mobile/i) != null || true;
+    }) ?? inputEls[0];
+    if (usernameEl && credentials.username) {
+      const uSelector = usernameEl.resourceId ? `id=${usernameEl.resourceId}` : `~${usernameEl.label}`;
+      try {
+        const uEl = await driver.$(uSelector);
+        await uEl.click();
+        await driver.pause(300);
+        await uEl.clearValue();
+        await uEl.setValue(credentials.username);
+      } catch {
+      }
+    }
+    const pwEl = elements.find(isPasswordField);
+    if (pwEl && credentials.password) {
+      const pSelector = pwEl.resourceId ? `id=${pwEl.resourceId}` : `~${pwEl.label}`;
+      try {
+        const pEl = await driver.$(pSelector);
+        await pEl.click();
+        await driver.pause(300);
+        await pEl.clearValue();
+        await pEl.setValue(credentials.password);
+      } catch {
+      }
+    }
+    try {
+      await driver.hideKeyboard();
+    } catch {
+    }
+    await driver.pause(500);
+    const loginLabel = credentials.loginButtonLabel ?? "";
+    const loginBtn = elements.find(
+      (e) => e.role === "button" && (loginLabel ? e.label?.toLowerCase().includes(loginLabel.toLowerCase()) : e.label?.toLowerCase().match(/log\s?in|sign\s?in|submit|continue|next|connect/i) != null)
+    );
+    if (loginBtn) {
+      const btnSelector = loginBtn.resourceId ? `id=${loginBtn.resourceId}` : `~${loginBtn.label}`;
+      try {
+        await driver.$(btnSelector).click();
+        await driver.pause(3e3);
+        logger.info({ platform }, "Native auth: login button tapped \u2014 waiting for app");
+        return true;
+      } catch {
+      }
+    }
+    return false;
+  } catch (err) {
+    logger.warn({ err }, "Native auth attempt failed \u2014 continuing without login");
+    return false;
+  }
+}
+async function performSwipe(driver, platform, direction) {
+  try {
+    const size = await driver.getWindowSize();
+    const { width: W, height: H } = size;
+    const midX = Math.round(W / 2);
+    const midY = Math.round(H / 2);
+    if (platform === "android") {
+      const startX = direction === "left" ? Math.round(W * 0.8) : direction === "right" ? Math.round(W * 0.2) : midX;
+      const startY = direction === "up" ? Math.round(H * 0.8) : direction === "down" ? Math.round(H * 0.2) : midY;
+      const endX = direction === "left" ? Math.round(W * 0.2) : direction === "right" ? Math.round(W * 0.8) : midX;
+      const endY = direction === "up" ? Math.round(H * 0.2) : direction === "down" ? Math.round(H * 0.8) : midY;
+      await driver.touchAction([
+        { action: "press", x: startX, y: startY },
+        { action: "wait", ms: 300 },
+        { action: "moveTo", x: endX, y: endY },
+        { action: "release" }
+      ]);
+    } else if (platform === "ios") {
+      await driver.execute("mobile: swipe", { direction });
+    } else {
+      await driver.execute("mobile: scroll", { direction });
+    }
+    await driver.pause(500);
+  } catch {
+  }
+}
+async function performLongPress(driver, selector, durationMs = 1200) {
+  try {
+    const el = await driver.$(selector);
+    await driver.touchAction([
+      { action: "longPress", element: el, ms: durationMs },
+      { action: "release" }
+    ]);
+    await driver.pause(600);
+  } catch {
+  }
+}
+async function visionFallbackDescribeScreen(screenshotBase64, llmConfig) {
+  try {
+    const Anthropic = (await import("@anthropic-ai/sdk")).default;
+    if (!llmConfig.apiKey) throw new Error("No LLM API key configured. Configure an Anthropic key in Workspace \u2192 Settings \u2192 AI Provider.");
+    const client = new Anthropic({ apiKey: llmConfig.apiKey });
+    const response = await client.messages.create({
+      model: llmConfig.model ?? "claude-haiku-4-5-20251001",
+      max_tokens: 512,
+      messages: [{
+        role: "user",
+        content: [
+          { type: "image", source: { type: "base64", media_type: "image/png", data: screenshotBase64 } },
+          { type: "text", text: 'This is a screenshot from a mobile app. The accessibility tree returned no elements. Describe what you see on screen in 1-2 sentences, then list ALL interactive UI elements (buttons, inputs, links, toggles) as JSON: [{"role":"button","label":"Sign In"},...]' }
+        ]
+      }]
+    });
+    const text = response.content[0]?.text ?? "";
+    const jsonMatch = text.match(/\[[\s\S]*\]/);
+    const elements = jsonMatch ? JSON.parse(jsonMatch[0]) : [];
+    return { screenDescription: text.split("[")[0]?.trim() ?? "", suggestedElements: elements };
+  } catch (err) {
+    logger.warn({ err }, "Vision fallback LLM call failed");
+  }
+  return { screenDescription: "Screen captured (accessibility tree empty \u2014 vision fallback unavailable)", suggestedElements: [] };
+}
+async function startMitmProxy(port) {
+  const captured = [];
+  try {
+    const net = await import("net");
+    const server = net.createServer((socket) => {
+      socket.once("data", (data) => {
+        const raw = data.toString("utf8", 0, 512);
+        const match = /^(GET|POST|PUT|DELETE|PATCH|HEAD)\s+(https?:\/\/[^\s]+)/i.exec(raw);
+        if (match) captured.push({ method: match[1], url: match[2] });
+        socket.destroy();
+      });
+    });
+    server.listen(port, "127.0.0.1");
+    const stop = () => server.close();
+    logger.info({ port }, "MITM proxy listener started (connection-level only \u2014 install device trust cert for full interception)");
+    return { captured, stop };
+  } catch (err) {
+    logger.warn({ err }, "MITM proxy start failed \u2014 API capture disabled");
+    return { captured, stop: () => {
+    } };
+  }
+}
+async function updateProgress(tenantId, jobId, progress, step) {
+  try {
+    await withTenant(
+      tenantId,
+      (tx) => tx.platformJob.update({
+        where: { id: jobId },
+        data: { progress, result: { step } }
+      })
+    );
+  } catch (err) {
+    logger.warn({ err }, "Failed to update job progress (non-fatal)");
+  }
+}
+async function runNativeCrawl(opts) {
+  const { tenantId, projectId, jobId, platform } = opts;
+  const screenshotDir = path.join(config.screenshotDir, projectId);
+  await fs.mkdir(screenshotDir, { recursive: true });
+  let tmpAppPath = null;
+  await withTenant(
+    tenantId,
+    (tx) => tx.platformJob.update({
+      where: { id: jobId },
+      data: { status: "RUNNING", startedAt: /* @__PURE__ */ new Date() }
+    })
+  );
+  const tenantStorageCfg = await getTenantStorageConfig(tenantId).catch(() => null);
+  try {
+    await updateProgress(tenantId, jobId, 5, "Downloading app binary");
+    tmpAppPath = await downloadAppFromS3(opts.appS3Path, platform, opts, tenantStorageCfg);
+    await updateProgress(tenantId, jobId, 10, "Building Appium session capabilities");
+    const capabilities = buildCapabilities(opts, tmpAppPath);
+    await updateProgress(tenantId, jobId, 15, "Starting Appium session");
+    const appiumServerUrl = opts.appiumServerUrl ?? process.env.APPIUM_SERVER_URL ?? "http://127.0.0.1:4723";
+    try {
+      const probe = await fetch(`${appiumServerUrl}/status`, { signal: AbortSignal.timeout(5e3) });
+      if (!probe.ok) throw new Error(`HTTP ${probe.status}`);
+    } catch (e) {
+      throw new Error(
+        `Appium server at ${appiumServerUrl} is not reachable (${e.message}). Start it with: cd /opt/zeta-ai && docker compose -f docker-compose.prod.yml -f docker-compose.appium.yml up -d appium`
+      );
+    }
+    const wdio = await import("webdriverio");
+    const driver = await wdio.remote({
+      hostname: new URL(appiumServerUrl).hostname,
+      port: parseInt(new URL(appiumServerUrl).port || "4723", 10),
+      path: "/",
+      capabilities,
+      logLevel: "warn"
+    });
+    logger.info({ platform, deviceFarm: opts.deviceFarm }, "Appium session started");
+    try {
+      const sessionId = driver.sessionId;
+      await withTenant(
+        tenantId,
+        (tx) => tx.platformJob.update({
+          where: { id: jobId },
+          data: { appiumSessionId: sessionId }
+        })
+      );
+      await updateProgress(tenantId, jobId, 20, "App launched \u2014 extracting initial screen");
+      let mitmStop = null;
+      const mitmCaptured = [];
+      if (opts.mitmProxyPort) {
+        const mitm = await startMitmProxy(opts.mitmProxyPort);
+        mitmStop = mitm.stop;
+        mitmCaptured.push(...mitm.captured);
+      }
+      await driver.pause(2e3);
+      if (opts.credentials?.username && opts.credentials?.password) {
+        const loggedIn = await attemptNativeLogin(driver, platform, opts.credentials);
+        if (loggedIn) await driver.pause(2500);
+      }
+      const MAX_SCREENS = 30;
+      let screensFound = 0;
+      const tappedSelectors = /* @__PURE__ */ new Set();
+      for (let i = 0; i < MAX_SCREENS; i++) {
+        const pct = 20 + Math.round(i / MAX_SCREENS * 60);
+        await updateProgress(tenantId, jobId, pct, `Capturing screen ${i + 1}`);
+        let screenName = `Screen ${i + 1}`;
+        try {
+          if (platform === "android") {
+            const activity = await driver.getCurrentActivity();
+            screenName = activity.split(".").pop() ?? screenName;
+          } else if (platform === "ios") {
+            const titleEl = await driver.$("~NavigationBarTitle").catch(() => null);
+            if (titleEl) {
+              const titleText = await titleEl.getText().catch(() => "");
+              if (titleText) screenName = titleText;
+            }
+          } else {
+            const winTitle = await driver.getTitle().catch(() => "");
+            if (winTitle) screenName = winTitle;
+          }
+        } catch {
+        }
+        const screenshotBase64 = await driver.takeScreenshot();
+        const screenshotBuffer = Buffer.from(screenshotBase64, "base64");
+        let screenshotBuf = screenshotBuffer;
+        try {
+          const sharp = (await import("sharp")).default;
+          const meta = await sharp(screenshotBuffer).metadata();
+          const logicalSize = await driver.getWindowSize().catch(() => ({ width: meta.width ?? 0, height: meta.height ?? 0 }));
+          const physW = meta.width ?? 0;
+          const logW = logicalSize.width;
+          if (physW > 0 && logW > 0 && physW > logW) {
+            const logH = Math.round((meta.height ?? physW) * (logW / physW));
+            screenshotBuf = Buffer.from(await sharp(screenshotBuffer).resize(logW, logH, { fit: "fill" }).png().toBuffer());
+          }
+        } catch {
+        }
+        const screen = await AppBrain.saveScreen(tenantId, projectId, screenName);
+        const shotPath = path.join(screenshotDir, `${screen.id}-native-v${screen.version}.png`);
+        await fs.writeFile(shotPath, screenshotBuf);
+        await AppBrain.saveScreenshot(tenantId, projectId, screen.id, shotPath, screen.version, "NATIVE");
+        if (tenantStorageCfg) {
+          const s3Key = `${tenantId}/${projectId}/screenshots/${screen.id}-native-v${screen.version}.png`;
+          await uploadScreenshotToTenantStorage(tenantStorageCfg, shotPath, s3Key).catch(() => {
+          });
+        }
+        try {
+          if (platform === "android") {
+            await driver.execute("mobile: scroll", { direction: "down" });
+          } else if (platform === "ios") {
+            await driver.execute("mobile: scroll", { direction: "down", element: null });
+          }
+          await driver.pause(400);
+        } catch {
+        }
+        const pageXml = await driver.getPageSource();
+        const nativeElements = parseAccessibilityTree(pageXml, platform);
+        try {
+          if (platform === "android") {
+            await driver.execute("mobile: scroll", { direction: "up" });
+          } else if (platform === "ios") {
+            await driver.execute("mobile: scroll", { direction: "up", element: null });
+          }
+          await driver.pause(300);
+        } catch {
+        }
+        logger.info({ screenName, elements: nativeElements.length }, "Native screen parsed");
+        if (nativeElements.length === 0 && opts.visionLlmConfig) {
+          logger.info({ screenName }, "Empty accessibility tree \u2014 triggering vision LLM fallback");
+          const { suggestedElements } = await visionFallbackDescribeScreen(screenshotBase64, opts.visionLlmConfig);
+          for (const el of suggestedElements) {
+            nativeElements.push({
+              role: el.role,
+              label: el.label,
+              resourceId: "",
+              className: "",
+              bounds: { x: 0, y: 0, width: 0, height: 0 },
+              platform
+            });
+          }
+        }
+        for (const el of nativeElements) {
+          const boundingRect = {
+            x: Math.round(el.bounds.x),
+            y: Math.round(el.bounds.y),
+            width: Math.round(el.bounds.width),
+            height: Math.round(el.bounds.height)
+          };
+          const notes = JSON.stringify({
+            ...el.resourceId ? { resourceId: el.resourceId } : {},
+            ...el.className ? { className: el.className } : {},
+            platform: el.platform
+          });
+          await AppBrain.saveElement(
+            tenantId,
+            projectId,
+            screen.id,
+            el.label,
+            // meaning
+            el.role,
+            void 0,
+            // expectedData
+            notes,
+            void 0,
+            // text
+            true,
+            // isInteractive
+            boundingRect,
+            void 0,
+            // ariaState
+            void 0,
+            // parentLandmark
+            "NATIVE"
+            // viewport
+          );
+        }
+        screensFound++;
+        const navElements = nativeElements.filter(
+          (el) => ["button", "text"].includes(el.role) || el.label?.match(/menu|home|back|next|continue|tab|nav|dashboard|settings|profile|more|list|search|help|about/i)
+        );
+        for (const navEl of navElements.slice(0, 15)) {
+          if (screensFound >= MAX_SCREENS) break;
+          try {
+            let selector;
+            if (platform === "android" && navEl.resourceId) {
+              selector = `id=${navEl.resourceId}`;
+            } else if (navEl.label) {
+              selector = `~${navEl.label}`;
+            } else {
+              continue;
+            }
+            if (tappedSelectors.has(selector)) continue;
+            tappedSelectors.add(selector);
+            let preActivity = "";
+            try {
+              if (platform === "android") {
+                preActivity = await driver.getCurrentActivity();
+              } else {
+                preActivity = await driver.getTitle();
+              }
+            } catch {
+            }
+            await driver.$(selector).click();
+            await driver.pause(1500);
+            if (platform === "android") {
+              try {
+                const logs = await driver.getLogs("logcat");
+                const crashLog = logs.find(
+                  (l) => typeof l.message === "string" && /FATAL EXCEPTION|Process.*has died|ANR in/i.test(l.message)
+                );
+                if (crashLog) {
+                  logger.error({ jobId, crash: String(crashLog.message).slice(0, 500) }, "App crash detected during crawl");
+                  break;
+                }
+              } catch {
+              }
+            }
+            if (platform === "ios") {
+              const crashLogs = await driver.getLogs("crashlog").catch(() => []);
+              if (crashLogs.length > 0 && crashLogs.some((l) => l.message?.includes("Exception") || l.message?.includes("crash"))) {
+                logger.warn({ screenName }, "iOS crash log detected \u2014 stopping crawl");
+                break;
+              }
+            }
+            let postActivity = "";
+            try {
+              if (platform === "android") {
+                postActivity = await driver.getCurrentActivity();
+              } else {
+                postActivity = await driver.getTitle();
+              }
+            } catch {
+            }
+            if (postActivity === preActivity) continue;
+            if (screensFound >= MAX_SCREENS) break;
+            const newScreenName = postActivity ? postActivity.split(".").pop() ?? `Screen ${screensFound + 1}` : `Screen ${screensFound + 1}`;
+            const newScreenshot = await driver.takeScreenshot();
+            const newScreen = await AppBrain.saveScreen(tenantId, projectId, newScreenName);
+            const newShotPath = path.join(screenshotDir, `${newScreen.id}-native-v${newScreen.version}.png`);
+            await fs.writeFile(newShotPath, Buffer.from(newScreenshot, "base64"));
+            await AppBrain.saveScreenshot(tenantId, projectId, newScreen.id, newShotPath, newScreen.version, "NATIVE");
+            if (tenantStorageCfg) {
+              const s3Key = `${tenantId}/${projectId}/screenshots/${newScreen.id}-native-v${newScreen.version}.png`;
+              await uploadScreenshotToTenantStorage(tenantStorageCfg, newShotPath, s3Key).catch(() => {
+              });
+            }
+            const newXml = await driver.getPageSource();
+            const newElements = parseAccessibilityTree(newXml, platform);
+            for (const el of newElements) {
+              await AppBrain.saveElement(
+                tenantId,
+                projectId,
+                newScreen.id,
+                el.label,
+                el.role,
+                void 0,
+                JSON.stringify({ ...el.resourceId ? { resourceId: el.resourceId } : {}, ...el.className ? { className: el.className } : {}, platform: el.platform }),
+                void 0,
+                true,
+                { x: Math.round(el.bounds.x), y: Math.round(el.bounds.y), width: Math.round(el.bounds.width), height: Math.round(el.bounds.height) },
+                void 0,
+                void 0,
+                "NATIVE"
+              );
+            }
+            screensFound++;
+            try {
+              await driver.back();
+              await driver.pause(800);
+            } catch {
+            }
+          } catch {
+          }
+        }
+        if (opts.gestureConfig?.enableSwipe && screensFound < MAX_SCREENS) {
+          for (const dir of ["left", "right", "up", "down"]) {
+            await performSwipe(driver, platform, dir);
+            const postSwipeXml = await driver.getPageSource().catch(() => "");
+            if (postSwipeXml && postSwipeXml !== pageXml) break;
+          }
+        }
+      }
+      if (mitmStop) mitmStop();
+      await driver.deleteSession();
+      await updateProgress(tenantId, jobId, 99, `Crawl complete \u2014 ${screensFound} screen(s) captured`);
+      await withTenant(
+        tenantId,
+        (tx) => tx.platformJob.update({
+          where: { id: jobId },
+          data: {
+            status: "DONE",
+            progress: 100,
+            finishedAt: /* @__PURE__ */ new Date(),
+            result: { screensFound, platform }
+          }
+        })
+      );
+      logger.info({ jobId, screensFound, platform }, "Native crawl complete");
+    } catch (sessionErr) {
+      try {
+        await driver.deleteSession();
+      } catch {
+      }
+      throw sessionErr;
+    }
+  } catch (err) {
+    logger.error({ jobId, err: String(err) }, "Native crawl failed");
+    await withTenant(
+      tenantId,
+      (tx) => tx.platformJob.update({
+        where: { id: jobId },
+        data: {
+          status: "FAILED",
+          error: String(err?.message ?? err),
+          finishedAt: /* @__PURE__ */ new Date()
+        }
+      })
+    ).catch(() => {
+    });
+    throw err;
+  } finally {
+    if (tmpAppPath) {
+      fs.unlink(tmpAppPath).catch(() => {
+      });
+    }
+  }
+}
+export {
+  runNativeCrawl
+};
+//# sourceMappingURL=native-crawler.js.map