@loadmill/droid-cua 2.19.0 → 2.22.0

package/build/src/integrations/loadmill/client.js

@@ -1,6 +1,7 @@
 /**
  * Loadmill API client for interacting with test flows
  */
+import { extractLoadmillSessionUser } from "./session-user.js";
 const DEFAULT_BASE_URL = "https://app.loadmill.com/api";
 function normalizeApiBaseUrl(rawBaseUrl) {
     const candidate = (rawBaseUrl || DEFAULT_BASE_URL).trim();
@@ -19,6 +20,10 @@ function normalizeApiBaseUrl(rawBaseUrl) {
         return DEFAULT_BASE_URL;
     }
 }
+function normalizeSiteBaseUrl(rawBaseUrl) {
+    const apiBaseUrl = normalizeApiBaseUrl(rawBaseUrl);
+    return apiBaseUrl.replace(/\/api$/, "");
+}
 /**
  * @param {{
  *   token?: string | null,
@@ -33,6 +38,9 @@ export function createLoadmillApiClient({ token = null, baseUrl = null, onUnauth
     function getBaseUrl() {
         return normalizeApiBaseUrl(baseUrl);
     }
+    function getSiteBaseUrl() {
+        return normalizeSiteBaseUrl(baseUrl);
+    }
     async function apiRequest(endpoint, options = {}) {
         const resolvedToken = getApiToken();
         if (!resolvedToken) {
@@ -124,9 +132,38 @@ export function createLoadmillApiClient({ token = null, baseUrl = null, onUnauth
             }),
         });
     }
+    async function reportDroidRun(payload) {
+        return await apiRequest(`/droid-cua/runs`, {
+            method: "POST",
+            body: JSON.stringify(payload),
+        });
+    }
+    async function getCurrentUser() {
+        const resolvedToken = getApiToken();
+        if (!resolvedToken) {
+            return null;
+        }
+        const response = await fetch(`${getSiteBaseUrl()}/auth/logged-in-user`, {
+            method: "GET",
+            headers: {
+                Accept: "application/json",
+                Authorization: `Bearer ${resolvedToken}`,
+            },
+        });
+        if (!response.ok) {
+            return null;
+        }
+        try {
+            return extractLoadmillSessionUser(await response.json());
+        }
+        catch {
+            return null;
+        }
+    }
     return {
         getApiToken,
         getBaseUrl,
+        getSiteBaseUrl,
         searchFlows,
         runTestSuite,
         runTestFlow,
@@ -134,6 +171,8 @@ export function createLoadmillApiClient({ token = null, baseUrl = null, onUnauth
         getFlowRunStatus,
         getTestSuiteFlowRun,
         explainFailure,
+        reportDroidRun,
+        getCurrentUser,
     };
 }
 export function getApiToken() {
@@ -185,6 +224,13 @@ export async function getFlowRunStatus(flowRunId) {
         onUnauthorized: globalThis.__DROID_CUA_HANDLE_LOADMILL_UNAUTHORIZED__,
     }).getFlowRunStatus(flowRunId);
 }
+export async function reportDroidRun(payload) {
+    return await createLoadmillApiClient({
+        token: process.env.LOADMILL_API_TOKEN,
+        baseUrl: process.env.LOADMILL_BASE_URL,
+        onUnauthorized: globalThis.__DROID_CUA_HANDLE_LOADMILL_UNAUTHORIZED__,
+    }).reportDroidRun(payload);
+}
 export async function getTestSuiteFlowRun(flowRunId) {
     return await createLoadmillApiClient({
         token: process.env.LOADMILL_API_TOKEN,

package/build/src/integrations/loadmill/executor.js

@@ -57,13 +57,13 @@ ${futureScript}`
         }
     ];
 }
-async function selectRelevantParameterNames({ openAiClient, command, flowName, futureInstructions, parameters, }) {
+async function selectRelevantParameterNames({ openAiClient, helperModel, command, flowName, futureInstructions, parameters, }) {
     if (!openAiClient || !Array.isArray(parameters) || parameters.length === 0) {
         return [];
     }
     try {
         const response = await openAiClient.chat.completions.create({
-            model: LOADMILL_MODEL_ALIASES.DEFAULT_AGENT_MODEL,
+            model: helperModel || LOADMILL_MODEL_ALIASES.DEFAULT_AGENT_MODEL,
             messages: buildParameterSelectionMessages({
                 command,
                 flowName,
@@ -98,7 +98,7 @@ function findSelectedSuiteFlowRun(finalResult, selectedFlow) {
     }
     return flowRuns.find((flowRun) => flowRun?.testSuiteFlowId === selectedFlow?.id) || null;
 }
-async function buildRuntimeParameterContext({ loadmillClient, finalResult, selectedFlow, command, flowName, futureInstructions, openAiClient, }) {
+async function buildRuntimeParameterContext({ loadmillClient, finalResult, selectedFlow, command, flowName, futureInstructions, openAiClient, helperModel, }) {
     const selectedSuiteFlowRun = findSelectedSuiteFlowRun(finalResult, selectedFlow);
     if (!selectedSuiteFlowRun?.id || typeof loadmillClient.getTestSuiteFlowRun !== "function") {
         return null;
@@ -112,6 +112,7 @@ async function buildRuntimeParameterContext({ loadmillClient, finalResult, selec
             flowName,
             futureInstructions,
             parameters,
+            helperModel,
         });
         const candidates = selectLoadmillParameterCandidates(parameters, selectedParameterNames);
         const note = renderLoadmillParameterContextNote(candidates);
@@ -198,7 +199,7 @@ async function pollForCompletion(runId, loadmillClient, options = {}) {
  * @returns {Promise<{success: boolean, error?: string, result?: Object}>}
  */
 export async function executeLoadmillCommand(userInput, options = {}) {
-    const { onProgress = () => { }, loadmillClient = createLoadmillApiClient(), openAiClient = null, futureInstructions = [], pollTimeoutMs, pollIntervalMs, now, sleepFn, } = options;
+    const { onProgress = () => { }, loadmillClient = createLoadmillApiClient(), openAiClient = null, helperModel = null, futureInstructions = [], pollTimeoutMs, pollIntervalMs, now, sleepFn, } = options;
     // Check for API token
     if (!loadmillClient.getApiToken()) {
         return {
@@ -215,7 +216,10 @@ export async function executeLoadmillCommand(userInput, options = {}) {
     try {
         // Step 1: Interpret the command
         onProgress({ step: "interpreting", message: "Interpreting command..." });
-        const interpreted = await interpretLoadmillCommand(userInput, { client: openAiClient });
+        const interpreted = await interpretLoadmillCommand(userInput, {
+            client: openAiClient,
+            ...(helperModel ? { model: helperModel } : {}),
+        });
         logger.debug("Loadmill command interpreted", interpreted);
         // Step 2: Search for flows
         onProgress({ step: "searching", message: `Searching for flows matching "${interpreted.searchQuery}"...` });
@@ -229,7 +233,10 @@ export async function executeLoadmillCommand(userInput, options = {}) {
         }
         // Step 3: Select best match
         onProgress({ step: "selecting", message: `Found ${flows.length} flow(s). Selecting best match...` });
-        const { selectedFlow, confidence } = await selectBestFlow(flows, interpreted.searchQuery, { client: openAiClient });
+        const { selectedFlow, confidence } = await selectBestFlow(flows, interpreted.searchQuery, {
+            client: openAiClient,
+            ...(helperModel ? { model: helperModel } : {}),
+        });
         if (!selectedFlow) {
             return {
                 success: false,
@@ -272,6 +279,7 @@ export async function executeLoadmillCommand(userInput, options = {}) {
                 flowName,
                 futureInstructions,
                 openAiClient,
+                helperModel,
             })
             : null;
         return {

package/build/src/integrations/loadmill/interpreter.js

@@ -8,9 +8,9 @@ import { buildLoadmillCommandInterpretationMessages, buildLoadmillFlowSelectionM
  * @param {string} userInput - Natural language command
  * @returns {Promise<{searchQuery: string, parameters: Object}>}
  */
-export async function interpretLoadmillCommand(userInput, { client }) {
+export async function interpretLoadmillCommand(userInput, { client, model = LOADMILL_MODEL_ALIASES.FAST }) {
     const response = await client.chat.completions.create({
-        model: LOADMILL_MODEL_ALIASES.FAST,
+        model,
         messages: buildLoadmillCommandInterpretationMessages(userInput),
         response_format: { type: "json_object" }
     });
@@ -27,7 +27,7 @@ export async function interpretLoadmillCommand(userInput, { client }) {
  * @param {string} originalQuery - Original user query
  * @returns {Promise<{selectedFlow: Object|null, confidence: number}>}
  */
-export async function selectBestFlow(flows, originalQuery, { client }) {
+export async function selectBestFlow(flows, originalQuery, { client, model = LOADMILL_MODEL_ALIASES.FAST }) {
     // Ensure flows is an array
     if (!flows || !Array.isArray(flows) || flows.length === 0) {
         return { selectedFlow: null, confidence: 0 };
@@ -42,7 +42,7 @@ export async function selectBestFlow(flows, originalQuery, { client }) {
         return `${i + 1}. ID: ${f.id}, Name: "${name}"${suite ? `, Suite: "${suite}"` : ""}`;
     }).join("\n");
     const response = await client.chat.completions.create({
-        model: LOADMILL_MODEL_ALIASES.FAST,
+        model,
         messages: buildLoadmillFlowSelectionMessages(originalQuery, flowList),
         response_format: { type: "json_object" }
     });

package/build/src/integrations/loadmill/run-reporter.js

@@ -0,0 +1,113 @@
+const VALID_STATUSES = new Set(["pass", "fail", "error", "skipped", "stopped"]);
+function trimToNull(value) {
+    if (typeof value !== "string") {
+        return null;
+    }
+    const trimmed = value.trim();
+    return trimmed || null;
+}
+function nonNegativeNumberOrNull(value) {
+    return typeof value === "number" && Number.isFinite(value) ? Math.max(0, value) : null;
+}
+function nonNegativeIntegerOrNull(value) {
+    const normalized = nonNegativeNumberOrNull(value);
+    return normalized == null ? null : Math.floor(normalized);
+}
+function normalizeSource(value) {
+    return value === "ci" ? "ci" : "desktop";
+}
+function normalizePlatform(value) {
+    const platform = trimToNull(value);
+    return platform === "browser" ? "web" : platform;
+}
+function normalizeTestName(value) {
+    return value ? value.replace(/\.dcua$/i, "") : null;
+}
+function normalizeRunType(value, testCases) {
+    if (value === "project" || value === "test") {
+        return value;
+    }
+    return testCases > 1 ? "project" : "test";
+}
+export function normalizeDroidRunStatus({ status = null, outcome = null, success = null, error = null } = {}) {
+    if (VALID_STATUSES.has(status)) {
+        return status;
+    }
+    if (outcome === "pass" || success === true) {
+        return "pass";
+    }
+    if (error === "Stopped by user") {
+        return "stopped";
+    }
+    if (outcome === "skipped") {
+        return "skipped";
+    }
+    if (outcome === "failure" || (typeof error === "string" && error.startsWith("Assertion failed:"))) {
+        return "fail";
+    }
+    return "error";
+}
+export function buildDroidRunReportPayload(input = {}) {
+    const completedAt = trimToNull(input.completedAt) || new Date().toISOString();
+    const testCases = nonNegativeIntegerOrNull(input.testCases);
+    const payload = {
+        runType: normalizeRunType(input.runType, testCases),
+        testCases,
+        durationMs: nonNegativeIntegerOrNull(input.durationMs),
+        platform: normalizePlatform(input.platform),
+        deviceSource: trimToNull(input.deviceSource),
+        deviceName: trimToNull(input.deviceName),
+        osVersion: trimToNull(input.osVersion),
+        instructionsTotal: nonNegativeIntegerOrNull(input.instructionsTotal),
+        instructionsCompleted: nonNegativeIntegerOrNull(input.instructionsCompleted),
+        actionsTotal: nonNegativeIntegerOrNull(input.actionsTotal),
+        assertionsPassed: nonNegativeIntegerOrNull(input.assertionsPassed),
+        assertionsFailed: nonNegativeIntegerOrNull(input.assertionsFailed),
+        retries: nonNegativeIntegerOrNull(input.retries),
+        error: trimToNull(input.error),
+    };
+    for (const key of Object.keys(payload)) {
+        if (payload[key] == null) {
+            delete payload[key];
+        }
+    }
+    return {
+        runId: trimToNull(input.runId),
+        source: normalizeSource(input.source),
+        projectName: trimToNull(input.projectName),
+        testName: normalizeTestName(trimToNull(input.testName)),
+        status: normalizeDroidRunStatus(input),
+        completedAt,
+        payload,
+    };
+}
+/**
+ * @param {{
+ *   loadmillClient?: { getApiToken?: () => string | null, reportDroidRun?: (payload: object) => Promise<unknown> } | null,
+ *   onWarning?: ((message: string) => void) | null,
+ *   [key: string]: unknown,
+ * }} input
+ */
+export async function reportDroidRunCompletion({ loadmillClient, onWarning = null, ...input } = {}) {
+    if (!loadmillClient?.getApiToken?.()) {
+        return { reported: false, reason: "missing_token" };
+    }
+    if (typeof loadmillClient.reportDroidRun !== "function") {
+        return { reported: false, reason: "unsupported_client" };
+    }
+    const payload = buildDroidRunReportPayload(input);
+    if (!payload.runId) {
+        return { reported: false, reason: "missing_run_id" };
+    }
+    try {
+        await loadmillClient.reportDroidRun(payload);
+        return { reported: true };
+    }
+    catch (error) {
+        const message = error instanceof Error ? error.message : String(error);
+        if (typeof onWarning === "function") {
+            onWarning(`Failed to report Droid run to Loadmill: ${message}`);
+        }
+        return { reported: false, reason: "request_failed", error: message };
+    }
+}

package/build/src/integrations/loadmill/session-user.js

@@ -0,0 +1,39 @@
+function trimToNull(value) {
+    if (typeof value !== "string") {
+        return null;
+    }
+    const trimmed = value.trim();
+    return trimmed || null;
+}
+function extractUserDisplayNameFromRecord(record) {
+    for (const key of ["displayName", "fullName", "name", "email"]) {
+        const value = trimToNull(record[key]);
+        if (value) {
+            return value;
+        }
+    }
+    const nested = record.user;
+    if (nested && typeof nested === "object") {
+        return extractUserDisplayNameFromRecord(nested);
+    }
+    return null;
+}
+export function extractLoadmillSessionUser(payload) {
+    if (!payload || typeof payload !== "object") {
+        return null;
+    }
+    const record = payload;
+    const id = trimToNull(record.id) || trimToNull(record.userId);
+    const email = trimToNull(record.email);
+    const displayName = extractUserDisplayNameFromRecord(record);
+    const teamId = trimToNull(record.teamId);
+    if (!id) {
+        return null;
+    }
+    return {
+        id,
+        email,
+        displayName,
+        teamId,
+    };
+}

package/build/src/integrations/segment/droid-cua-segment.js

@@ -0,0 +1,64 @@
+import { DroidCuaSegmentEvent, buildDroidCuaTestRunCompletedProperties, } from "../../core/droid-cua-segment-events.js";
+const DEFAULT_DROID_CUA_SEGMENT_WRITE_KEY = "ZuHGZkEXjhEkvzVPMnVDJfWiNsD1nGhG";
+const SEGMENT_TRACK_URL = "https://api.segment.io/v1/track";
+function trimToNull(value) {
+    if (typeof value !== "string") {
+        return null;
+    }
+    const trimmed = value.trim();
+    return trimmed || null;
+}
+function getSegmentWriteKey() {
+    if (process.env.DROID_CUA_SEGMENT_DISABLED === "1") {
+        return null;
+    }
+    if (process.env.DROID_CUA_SEGMENT_DISABLED === "true") {
+        return null;
+    }
+    return trimToNull(process.env.SEGMENT_WRITE_KEY) || DEFAULT_DROID_CUA_SEGMENT_WRITE_KEY;
+}
+function buildBasicAuthHeader(writeKey) {
+    return `Basic ${Buffer.from(`${writeKey}:`).toString("base64")}`;
+}
+function warnSegmentError(scope, error) {
+    console.warn(`[segment] ${scope}`, error);
+}
+export async function reportDroidCuaCiTestRunCompleted(input = {}) {
+    const writeKey = getSegmentWriteKey();
+    if (!writeKey) {
+        return { reported: false, reason: "disabled" };
+    }
+    const userId = trimToNull(input.userId);
+    const anonymousId = trimToNull(input.anonymousId) || (input.runId ? `droid-cua-ci:${input.runId}` : null);
+    if (!userId && !anonymousId) {
+        return { reported: false, reason: "missing_identity" };
+    }
+    const body = {
+        ...(userId ? { userId } : { anonymousId }),
+        event: DroidCuaSegmentEvent.TEST_RUN_COMPLETED,
+        properties: buildDroidCuaTestRunCompletedProperties(input),
+    };
+    try {
+        const response = await fetch(SEGMENT_TRACK_URL, {
+            method: "POST",
+            headers: {
+                Authorization: buildBasicAuthHeader(writeKey),
+                "Content-Type": "application/json",
+            },
+            body: JSON.stringify(body),
+        });
+        if (!response.ok) {
+            const detail = await response.text();
+            return { reported: false, reason: "request_failed", error: `Segment API error (${response.status}): ${detail}` };
+        }
+        return { reported: true };
+    }
+    catch (error) {
+        warnSegmentError("CI test run completed reporting failed", error);
+        return {
+            reported: false,
+            reason: "request_failed",
+            error: error instanceof Error ? error.message : String(error),
+        };
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loadmill/droid-cua",
-  "version": "2.19.0",
+  "version": "2.22.0",
   "description": "AI-powered Android testing agent using OpenAI's computer-use model and ADB",
   "main": "build/index.js",
   "type": "module",
@@ -20,8 +20,11 @@
     "bench:run": "tsx index.js --platform android --instructions benchmarks/benchmark.dcua --context benchmarks/context.md --debug",
     "bench:report": "node scripts/benchmark-report.js",
     "bench:report:web": "node scripts/benchmark-report-web.js",
+    "obs:droid": "node scripts/obs-droid-demo.mjs",
     "execution:profile": "node scripts/execution-profile.js",
     "test": "node --test unit-tests/*.test.js",
+    "test:aws-device-farm:android": "node integration-tests/aws-device-farm-android.test.js",
+    "test:aws-device-farm:ios": "node integration-tests/aws-device-farm-ios.test.js",
     "build": "tsc",
     "clean": "rm -rf build",
     "typecheck": "tsc --noEmit",
@@ -58,6 +61,7 @@
   "author": "Loadmill",
   "license": "UNLICENSED",
   "dependencies": {
+    "@aws-sdk/client-device-farm": "^3.1065.0",
     "dotenv": "^16.4.7",
     "ink": "^4.4.1",
     "ink-spinner": "^5.0.0",