agent-device 0.12.3 → 0.12.5

package/dist/src/testing/conformance.d.ts

@@ -0,0 +1,416 @@
+declare type AgentDeviceBackend = {
+    platform: AgentDeviceBackendPlatform;
+    capabilities?: BackendCapabilitySet;
+    escapeHatches?: BackendEscapeHatches;
+    captureSnapshot?(context: BackendCommandContext, options?: BackendSnapshotOptions): Promise<BackendSnapshotResult>;
+    captureScreenshot?(context: BackendCommandContext, outPath: string, options?: BackendScreenshotOptions): Promise<BackendScreenshotResult | void>;
+    readText?(context: BackendCommandContext, node: SnapshotNode): Promise<BackendReadTextResult>;
+    findText?(context: BackendCommandContext, text: string): Promise<BackendFindTextResult>;
+    tap?(context: BackendCommandContext, point: Point, options?: BackendTapOptions): Promise<BackendActionResult>;
+    fill?(context: BackendCommandContext, point: Point, text: string, options?: BackendFillOptions): Promise<BackendActionResult>;
+    typeText?(context: BackendCommandContext, text: string, options?: {
+        delayMs?: number;
+    }): Promise<BackendActionResult>;
+    pressKey?(context: BackendCommandContext, key: string, options?: {
+        modifiers?: string[];
+    }): Promise<BackendActionResult>;
+    openApp?(context: BackendCommandContext, target: BackendOpenTarget): Promise<BackendActionResult>;
+    closeApp?(context: BackendCommandContext, app?: string): Promise<BackendActionResult>;
+    installApp?(context: BackendCommandContext, target: BackendInstallTarget): Promise<BackendActionResult>;
+};
+
+declare type AgentDeviceBackendPlatform = 'ios' | 'android' | 'macos' | 'linux';
+
+declare type AgentDeviceRuntime = {
+    backend: AgentDeviceBackend;
+    artifacts: ArtifactAdapter;
+    sessions: CommandSessionStore;
+    policy: CommandPolicy;
+    diagnostics?: DiagnosticsSink;
+    clock?: CommandClock;
+    signal?: AbortSignal;
+};
+
+declare type ArtifactAdapter = {
+    resolveInput(ref: FileInputRef, options: ResolveInputOptions): Promise<ResolvedInputFile>;
+    reserveOutput(ref: FileOutputRef | undefined, options: ReserveOutputOptions): Promise<ReservedOutputFile>;
+    createTempFile(options: CreateTempFileOptions): Promise<TemporaryFile>;
+};
+
+declare type ArtifactDescriptor = {
+    kind: 'localPath';
+    field: string;
+    path: string;
+    fileName?: string;
+    metadata?: Record<string, unknown>;
+} | {
+    kind: 'artifact';
+    field: string;
+    artifactId: string;
+    fileName?: string;
+    url?: string;
+    clientPath?: string;
+    metadata?: Record<string, unknown>;
+};
+
+export declare function assertCommandConformance(target: CommandConformanceTarget, options?: {
+    suites?: readonly CommandConformanceSuite[];
+}): Promise<CommandConformanceReport>;
+
+declare const BACKEND_CAPABILITY_NAMES: readonly ["android.shell", "ios.runnerCommand", "macos.desktopScreenshot"];
+
+declare type BackendActionResult = Record<string, unknown> | void;
+
+declare type BackendCapabilityName = (typeof BACKEND_CAPABILITY_NAMES)[number];
+
+declare type BackendCapabilitySet = readonly BackendCapabilityName[];
+
+declare type BackendCommandContext = {
+    session?: string;
+    requestId?: string;
+    appId?: string;
+    appBundleId?: string;
+    signal?: AbortSignal;
+    metadata?: Record<string, unknown>;
+};
+
+declare type BackendEscapeHatches = {
+    androidShell?(context: BackendCommandContext, args: readonly string[]): Promise<BackendShellResult>;
+    iosRunnerCommand?(context: BackendCommandContext, command: BackendRunnerCommand): Promise<BackendActionResult>;
+    macosDesktopScreenshot?(context: BackendCommandContext, outPath: string, options?: BackendScreenshotOptions): Promise<BackendScreenshotResult | void>;
+};
+
+declare type BackendFillOptions = {
+    delayMs?: number;
+};
+
+declare type BackendFindTextResult = {
+    found: boolean;
+};
+
+declare type BackendInstallTarget = {
+    app: string;
+    artifactPath: string;
+};
+
+declare type BackendOpenTarget = {
+    app?: string;
+    url?: string;
+    activity?: string;
+};
+
+declare type BackendReadTextResult = {
+    text: string;
+};
+
+declare type BackendRunnerCommand = {
+    command: string;
+    args?: readonly string[];
+    payload?: Record<string, unknown>;
+};
+
+declare type BackendScreenshotOptions = {
+    fullscreen?: boolean;
+    overlayRefs?: boolean;
+    surface?: 'app' | 'frontmost-app' | 'desktop' | 'menubar';
+};
+
+declare type BackendScreenshotResult = {
+    path?: string;
+    overlayRefs?: ScreenshotOverlayRef[];
+};
+
+declare type BackendShellResult = {
+    exitCode: number;
+    stdout: string;
+    stderr: string;
+};
+
+declare type BackendSnapshotAnalysis = {
+    rawNodeCount?: number;
+    maxDepth?: number;
+};
+
+declare type BackendSnapshotFreshness = {
+    action: string;
+    retryCount: number;
+    staleAfterRetries: boolean;
+    reason?: 'empty-interactive' | 'sharp-drop' | 'stuck-route';
+};
+
+declare type BackendSnapshotOptions = SnapshotOptions & {
+    outPath?: string;
+};
+
+declare type BackendSnapshotResult = {
+    nodes?: SnapshotNode[];
+    truncated?: boolean;
+    backend?: string;
+    snapshot?: SnapshotState;
+    analysis?: BackendSnapshotAnalysis;
+    freshness?: BackendSnapshotFreshness;
+    warnings?: string[];
+    appName?: string;
+    appBundleId?: string;
+};
+
+declare type BackendTapOptions = {
+    button?: 'primary' | 'secondary' | 'middle';
+    count?: number;
+    intervalMs?: number;
+    holdMs?: number;
+    jitterPx?: number;
+    doubleTap?: boolean;
+};
+
+export declare const captureConformanceSuite: CommandConformanceSuite;
+
+declare type CommandClock = {
+    now(): number;
+    sleep(ms: number): Promise<void>;
+};
+
+export declare type CommandConformanceCase = {
+    name: string;
+    command: string;
+    run(runtime: AgentDeviceRuntime, fixtures: CommandConformanceFixtures): Promise<void>;
+};
+
+export declare type CommandConformanceCaseContext = {
+    suite: string;
+    caseName: string;
+    fixtures: CommandConformanceFixtures;
+};
+
+export declare type CommandConformanceFailure = {
+    suite: string;
+    caseName: string;
+    command: string;
+    error: unknown;
+};
+
+export declare type CommandConformanceFixtures = {
+    session: string;
+    visibleSelector: string;
+    visibleText: string;
+    editableTarget: InteractionTarget;
+    fillText: string;
+    point: Point;
+};
+
+export declare type CommandConformanceReport = {
+    target: string;
+    passed: number;
+    failed: number;
+    failures: CommandConformanceFailure[];
+    suites: CommandConformanceSuiteResult[];
+};
+
+export declare type CommandConformanceSuite = {
+    name: string;
+    cases: readonly CommandConformanceCase[];
+    run(target: CommandConformanceTarget): Promise<CommandConformanceSuiteResult>;
+};
+
+export declare type CommandConformanceSuiteResult = {
+    suite: string;
+    passed: number;
+    failed: number;
+    failures: CommandConformanceFailure[];
+};
+
+export declare const commandConformanceSuites: readonly CommandConformanceSuite[];
+
+export declare type CommandConformanceTarget = {
+    name: string;
+    createRuntime: ConformanceRuntimeFactory;
+    fixtures?: Partial<CommandConformanceFixtures>;
+    beforeEach?(context: CommandConformanceCaseContext): void | Promise<void>;
+    afterEach?(context: CommandConformanceCaseContext): void | Promise<void>;
+};
+
+declare type CommandPolicy = {
+    allowLocalInputPaths: boolean;
+    allowLocalOutputPaths: boolean;
+    maxImagePixels: number;
+    allowNamedBackendCapabilities: readonly BackendCapabilityName[];
+};
+
+declare type CommandSessionRecord = {
+    name: string;
+    appId?: string;
+    appBundleId?: string;
+    appName?: string;
+    backendSessionId?: string;
+    snapshot?: SnapshotState;
+    metadata?: Record<string, unknown>;
+};
+
+declare type CommandSessionStore = {
+    get(name: string): CommandSessionRecord | undefined | Promise<CommandSessionRecord | undefined>;
+    set(record: CommandSessionRecord): void | Promise<void>;
+    delete?(name: string): void | Promise<void>;
+    list?(): readonly CommandSessionRecord[] | Promise<readonly CommandSessionRecord[]>;
+};
+
+export declare type ConformanceRuntimeFactory = () => AgentDeviceRuntime | Promise<AgentDeviceRuntime>;
+
+declare type CreateTempFileOptions = {
+    prefix: string;
+    ext: string;
+};
+
+export declare const defaultCommandConformanceFixtures: CommandConformanceFixtures;
+
+declare type DiagnosticsSink = {
+    emit(event: {
+        level: 'debug' | 'info' | 'warn' | 'error';
+        message: string;
+        data?: unknown;
+    }): void;
+};
+
+declare type ElementTarget = SelectorTarget | RefTarget;
+
+declare type FileInputRef = {
+    kind: 'path';
+    path: string;
+} | {
+    kind: 'uploadedArtifact';
+    id: string;
+};
+
+declare type FileOutputRef = {
+    kind: 'path';
+    path: string;
+} | {
+    kind: 'downloadableArtifact';
+    clientPath?: string;
+    fileName?: string;
+};
+
+export declare const interactionConformanceSuite: CommandConformanceSuite;
+
+declare type InteractionTarget = ElementTarget | PointTarget;
+
+declare type OutputVisibility = 'client-visible' | 'internal';
+
+declare type Point = {
+    x: number;
+    y: number;
+};
+
+declare type PointTarget = {
+    kind: 'point';
+    x: number;
+    y: number;
+};
+
+declare type RawSnapshotNode = {
+    index: number;
+    type?: string;
+    role?: string;
+    subrole?: string;
+    label?: string;
+    value?: string;
+    identifier?: string;
+    rect?: Rect;
+    enabled?: boolean;
+    selected?: boolean;
+    hittable?: boolean;
+    depth?: number;
+    parentIndex?: number;
+    pid?: number;
+    bundleId?: string;
+    appName?: string;
+    windowTitle?: string;
+    surface?: string;
+    hiddenContentAbove?: boolean;
+    hiddenContentBelow?: boolean;
+};
+
+declare type Rect = {
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+};
+
+declare type RefTarget = {
+    kind: 'ref';
+    ref: string;
+    fallbackLabel?: string;
+};
+
+declare type ReservedOutputFile = {
+    path: string;
+    visibility: OutputVisibility;
+    publish: () => Promise<ArtifactDescriptor | undefined>;
+    cleanup?: () => Promise<void>;
+};
+
+declare type ReserveOutputOptions = {
+    field: string;
+    ext: string;
+    requestedClientPath?: string;
+    visibility?: OutputVisibility;
+};
+
+declare type ResolvedInputFile = {
+    path: string;
+    cleanup?: () => Promise<void>;
+};
+
+declare type ResolveInputOptions = {
+    usage: string;
+    field?: string;
+};
+
+export declare function runCommandConformance(target: CommandConformanceTarget, options?: {
+    suites?: readonly CommandConformanceSuite[];
+}): Promise<CommandConformanceReport>;
+
+declare type ScreenshotOverlayRef = {
+    ref: string;
+    label?: string;
+    rect: Rect;
+    overlayRect: Rect;
+    center: Point;
+};
+
+export declare const selectorConformanceSuite: CommandConformanceSuite;
+
+declare type SelectorTarget = {
+    kind: 'selector';
+    selector: string;
+};
+
+declare type SnapshotBackend = 'xctest' | 'android' | 'macos-helper' | 'linux-atspi';
+
+declare type SnapshotNode = RawSnapshotNode & {
+    ref: string;
+};
+
+declare type SnapshotOptions = {
+    interactiveOnly?: boolean;
+    compact?: boolean;
+    depth?: number;
+    scope?: string;
+    raw?: boolean;
+};
+
+declare type SnapshotState = {
+    nodes: SnapshotNode[];
+    createdAt: number;
+    truncated?: boolean;
+    backend?: SnapshotBackend;
+    comparisonSafe?: boolean;
+};
+
+declare type TemporaryFile = {
+    path: string;
+    visibility: 'internal';
+    cleanup: () => Promise<void>;
+};
+
+export { }

package/dist/src/testing/conformance.js

@@ -0,0 +1 @@
+import e from"node:assert/strict";import{commands as t,selector as s}from"../commands/index.js";let a={session:"default",visibleSelector:"label=Continue",visibleText:"Continue",editableTarget:s("label=Email"),fillText:"hello@example.com",point:{x:4,y:8}},n=m({name:"capture",cases:[{name:"captures screenshots through the backend primitive",command:"capture.screenshot",run:async(s,a)=>{let n=await t.capture.screenshot(s,{session:a.session});e.equal(typeof n.path,"string"),e.ok(n.path.length>0)}},{name:"captures snapshots with nodes",command:"capture.snapshot",run:async(s,a)=>{let n=await t.capture.snapshot(s,{session:a.session});e.ok(Array.isArray(n.nodes))}}]}),i=m({name:"selectors",cases:[{name:"finds visible text",command:"selectors.find",run:async(s,a)=>{let n=await t.selectors.find(s,{session:a.session,query:a.visibleText,action:"exists"});e.equal(n.kind,"found"),e.equal(n.found,!0)}},{name:"reads text from a selector",command:"selectors.getText",run:async(a,n)=>{let i=await t.selectors.getText(a,{session:n.session,target:s(n.visibleSelector)});e.equal(i.kind,"text"),e.equal(i.text,n.visibleText)}},{name:"checks selector visibility",command:"selectors.isVisible",run:async(a,n)=>{let i=await t.selectors.isVisible(a,{session:n.session,target:s(n.visibleSelector)});e.equal(i.pass,!0)}},{name:"waits for visible text",command:"selectors.waitForText",run:async(s,a)=>{let n=await t.selectors.waitForText(s,{session:a.session,text:a.visibleText,timeoutMs:1});e.equal(n.kind,"text"),e.equal(n.text,a.visibleText)}}]}),o=m({name:"interactions",cases:[{name:"clicks selector targets",command:"interactions.click",run:async(a,n)=>{let i=await t.interactions.click(a,{session:n.session,target:s(n.visibleSelector)});e.equal(i.kind,"selector")}},{name:"presses explicit points",command:"interactions.press",run:async(s,a)=>{let n=await t.interactions.press(s,{session:a.session,target:{kind:"point",...a.point}});e.deepEqual(n.point,a.point)}},{name:"fills editable targets",command:"interactions.fill",run:async(s,a)=>{let n=await t.interactions.fill(s,{session:a.session,target:a.editableTarget,text:a.fillText});e.equal(n.text,a.fillText)}},{name:"types text without a target",command:"interactions.typeText",run:async(s,a)=>{let n=await t.interactions.typeText(s,{session:a.session,text:a.fillText});e.equal(n.text,a.fillText)}}]}),r=[n,i,o];async function l(e,t={}){let s=t.suites??r,a=[];for(let t of s)a.push(await t.run(e));let n=a.flatMap(e=>e.failures);return{target:e.name,passed:a.reduce((e,t)=>e+t.passed,0),failed:a.reduce((e,t)=>e+t.failed,0),failures:n,suites:a}}async function c(e,t={}){let s=await l(e,t);if(s.failed>0)throw AggregateError(s.failures.map(e=>e.error),`${e.name} failed ${s.failed} agent-device conformance case${1===s.failed?"":"s"}`);return s}function m(e){return{name:e.name,cases:e.cases,run:async t=>{let s={...a,...t.fixtures},n=[],i=0;for(let a of e.cases){let o={suite:e.name,caseName:a.name,fixtures:s};try{await t.beforeEach?.(o);let e=await t.createRuntime();await a.run(e,s),i+=1}catch(t){n.push({suite:e.name,caseName:a.name,command:a.command,error:t})}finally{await t.afterEach?.(o)}}return{suite:e.name,passed:i,failed:n.length,failures:n}}}}export{c as assertCommandConformance,n as captureConformanceSuite,r as commandConformanceSuites,a as defaultCommandConformanceFixtures,o as interactionConformanceSuite,l as runCommandConformance,i as selectorConformanceSuite};

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift

@@ -183,16 +183,25 @@ extension RunnerTests {
       if let requestedFps = command.fps, (requestedFps < minRecordingFps || requestedFps > maxRecordingFps) {
         return Response(ok: false, error: ErrorPayload(message: "recordStart fps must be between \(minRecordingFps) and \(maxRecordingFps)"))
       }
+      if let requestedQuality = command.quality, (requestedQuality < minRecordingQuality || requestedQuality > maxRecordingQuality) {
+        return Response(ok: false, error: ErrorPayload(message: "recordStart quality must be between \(minRecordingQuality) and \(maxRecordingQuality)"))
+      }
       do {
         let resolvedOutPath = resolveRecordingOutPath(requestedOutPath)
         let fpsLabel = command.fps.map(String.init) ?? String(RunnerTests.defaultRecordingFps)
+        let qualityLabel = command.quality.map(String.init) ?? "native"
         NSLog(
-          "AGENT_DEVICE_RUNNER_RECORD_START requestedOutPath=%@ resolvedOutPath=%@ fps=%@",
+          "AGENT_DEVICE_RUNNER_RECORD_START requestedOutPath=%@ resolvedOutPath=%@ fps=%@ quality=%@",
           requestedOutPath,
           resolvedOutPath,
-          fpsLabel
+          fpsLabel,
+          qualityLabel
+        )
+        let recorder = ScreenRecorder(
+          outputPath: resolvedOutPath,
+          fps: command.fps.map { Int32($0) },
+          quality: command.quality
         )
-        let recorder = ScreenRecorder(outputPath: resolvedOutPath, fps: command.fps.map { Int32($0) })
         try recorder.start { [weak self] in
           return self?.captureRunnerFrame()
         }

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift

@@ -52,6 +52,7 @@ struct Command: Codable {
   let scale: Double?
   let outPath: String?
   let fps: Int?
+  let quality: Int?
   let interactiveOnly: Bool?
   let compact: Bool?
   let depth: Int?

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScreenRecorder.swift

@@ -7,6 +7,7 @@ extension RunnerTests {
   final class ScreenRecorder {
     private let outputPath: String
     private let fps: Int32?
+    private let quality: Int?
     private var effectiveFps: Int32 {
       max(1, fps ?? RunnerTests.defaultRecordingFps)
     }
@@ -25,9 +26,10 @@ extension RunnerTests {
     private var startedSession = false
     private var startError: Error?
 
-    init(outputPath: String, fps: Int32?) {
+    init(outputPath: String, fps: Int32?, quality: Int?) {
       self.outputPath = outputPath
       self.fps = fps
+      self.quality = quality
     }
 
     func start(captureFrame: @escaping () -> RunnerImage?) throws {
@@ -48,7 +50,7 @@ extension RunnerTests {
       while Date() < bootstrapDeadline {
         if let image = captureFrame(), let cgImage = runnerCGImage(from: image) {
           bootstrapImage = image
-          dimensions = CGSize(width: cgImage.width, height: cgImage.height)
+          dimensions = scaledDimensions(width: cgImage.width, height: cgImage.height)
           break
         }
         Thread.sleep(forTimeInterval: 0.05)
@@ -240,11 +242,13 @@ extension RunnerTests {
 
       CVPixelBufferLockBaseAddress(pixelBuffer, [])
       defer { CVPixelBufferUnlockBaseAddress(pixelBuffer, []) }
+      let width = CVPixelBufferGetWidth(pixelBuffer)
+      let height = CVPixelBufferGetHeight(pixelBuffer)
       guard
         let context = CGContext(
           data: CVPixelBufferGetBaseAddress(pixelBuffer),
-          width: image.width,
-          height: image.height,
+          width: width,
+          height: height,
           bitsPerComponent: 8,
           bytesPerRow: CVPixelBufferGetBytesPerRow(pixelBuffer),
           space: CGColorSpaceCreateDeviceRGB(),
@@ -253,8 +257,23 @@ extension RunnerTests {
       else {
         return nil
       }
-      context.draw(image, in: CGRect(x: 0, y: 0, width: image.width, height: image.height))
+      context.draw(image, in: CGRect(x: 0, y: 0, width: width, height: height))
       return pixelBuffer
     }
+
+    private func scaledDimensions(width: Int, height: Int) -> CGSize {
+      guard let quality, quality < 10 else {
+        return CGSize(width: width, height: height)
+      }
+      let scale = Double(quality) / 10.0
+      return CGSize(
+        width: scaledEvenDimension(width, scale: scale),
+        height: scaledEvenDimension(height, scale: scale)
+      )
+    }
+
+    private func scaledEvenDimension(_ value: Int, scale: Double) -> Int {
+      max(2, Int((Double(value) * scale / 2.0).rounded()) * 2)
+    }
   }
 }

package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift

@@ -48,6 +48,8 @@ final class RunnerTests: XCTestCase {
   let tvRemoteDoublePressDelayDefault: TimeInterval = 0.0
   let minRecordingFps = 1
   let maxRecordingFps = 120
+  let minRecordingQuality = 5
+  let maxRecordingQuality = 10
   var needsPostSnapshotInteractionDelay = false
   var needsFirstInteractionDelay = false
   var activeRecording: ScreenRecorder?