agent-device 0.10.0 → 0.10.2

package/dist/src/daemon/handlers/interaction-common.d.ts

@@ -1,6 +1,6 @@
 import { dispatchCommand, type CommandFlags } from '../../core/dispatch.ts';
 import type { DaemonCommandContext } from '../context.ts';
-import type { DaemonRequest } from '../types.ts';
+import type { DaemonRequest, DaemonResponse, SessionState } from '../types.ts';
 import { SessionStore } from '../session-store.ts';
 export type ContextFromFlags = (flags: CommandFlags | undefined, appBundleId?: string, traceLogPath?: string) => DaemonCommandContext;
 export type InteractionHandlerParams = {
@@ -10,3 +10,32 @@ export type InteractionHandlerParams = {
     contextFromFlags: ContextFromFlags;
     dispatch: typeof dispatchCommand;
 };
+export declare function buildTouchVisualizationResult(params: {
+    data: Record<string, unknown> | undefined;
+    fallbackX: number;
+    fallbackY: number;
+    referenceFrame?: {
+        referenceWidth: number;
+        referenceHeight: number;
+    };
+    extra?: Record<string, unknown>;
+}): Record<string, unknown>;
+export declare function dispatchRecordedTouchInteraction(params: {
+    session: SessionState;
+    sessionStore: SessionStore;
+    requestCommand: string;
+    requestPositionals: string[];
+    flags: CommandFlags | undefined;
+    contextFromFlags: ContextFromFlags;
+    dispatch: typeof dispatchCommand;
+    interactionCommand: string;
+    interactionPositionals: string[];
+    outPath: string | undefined;
+    buildPayloads: (data: Record<string, unknown> | undefined) => {
+        result: Record<string, unknown>;
+        responseData?: Record<string, unknown>;
+    } | Promise<{
+        result: Record<string, unknown>;
+        responseData?: Record<string, unknown>;
+    }>;
+}): Promise<DaemonResponse>;

package/dist/src/daemon/handlers/interaction-read.d.ts

@@ -0,0 +1,14 @@
+import { dispatchCommand, type CommandFlags } from '../../core/dispatch.ts';
+import type { SessionState } from '../types.ts';
+import type { SnapshotNode } from '../../utils/snapshot.ts';
+import type { ContextFromFlags } from './interaction-common.ts';
+export declare function readTextForNode(params: {
+    device: SessionState['device'];
+    node: SnapshotNode;
+    flags: CommandFlags | undefined;
+    appBundleId?: string;
+    traceOutPath?: string;
+    surface?: SessionState['surface'];
+    contextFromFlags: ContextFromFlags;
+    dispatch: typeof dispatchCommand;
+}): Promise<string>;

package/dist/src/daemon/handlers/interaction-touch.d.ts

@@ -0,0 +1,45 @@
+import { dispatchCommand, type CommandFlags } from '../../core/dispatch.ts';
+import { type SnapshotNode } from '../../utils/snapshot.ts';
+import type { DaemonRequest, DaemonResponse, SessionState } from '../types.ts';
+import { SessionStore } from '../session-store.ts';
+import { getAndroidScreenSize } from '../../platforms/android/index.ts';
+import { type ContextFromFlags } from './interaction-common.ts';
+type CaptureSnapshotForSession = (session: SessionState, flags: CommandFlags | undefined, sessionStore: SessionStore, contextFromFlags: ContextFromFlags, options: {
+    interactiveOnly: boolean;
+}, dispatch?: typeof dispatchCommand) => Promise<{
+    nodes: SnapshotNode[];
+    truncated?: boolean;
+    createdAt: number;
+    backend?: 'xctest' | 'android' | 'macos-helper';
+}>;
+type ResolveRefTarget = ((params: {
+    session: SessionState;
+    refInput: string;
+    fallbackLabel: string;
+    requireRect: boolean;
+    invalidRefMessage: string;
+    notFoundMessage: string;
+}) => {
+    ok: true;
+    target: {
+        ref: string;
+        node: SnapshotNode;
+        snapshotNodes: SnapshotNode[];
+    };
+} | {
+    ok: false;
+    response: DaemonResponse;
+}) | undefined;
+type RefSnapshotFlagGuardResponse = (command: 'press' | 'fill' | 'get' | 'scrollintoview', flags: CommandFlags | undefined) => DaemonResponse | null;
+export declare function handleTouchInteractionCommands(params: {
+    req: DaemonRequest;
+    sessionName: string;
+    sessionStore: SessionStore;
+    contextFromFlags: ContextFromFlags;
+    dispatch?: typeof dispatchCommand;
+    readAndroidScreenSize?: typeof getAndroidScreenSize;
+    captureSnapshotForSession: CaptureSnapshotForSession;
+    resolveRefTarget: NonNullable<ResolveRefTarget>;
+    refSnapshotFlagGuardResponse: RefSnapshotFlagGuardResponse;
+}): Promise<DaemonResponse | null>;
+export {};

package/dist/src/daemon/handlers/interaction.d.ts

@@ -1,7 +1,9 @@
 import { dispatchCommand } from '../../core/dispatch.ts';
 import type { DaemonResponse } from '../types.ts';
+import { getAndroidScreenSize } from '../../platforms/android/index.ts';
 import type { InteractionHandlerParams } from './interaction-common.ts';
 export { unsupportedRefSnapshotFlags } from './interaction-flags.ts';
 export declare function handleInteractionCommands(params: Omit<InteractionHandlerParams, 'dispatch'> & {
     dispatch?: typeof dispatchCommand;
+    readAndroidScreenSize?: typeof getAndroidScreenSize;
 }): Promise<DaemonResponse | null>;

package/dist/src/daemon/handlers/record-trace-android.d.ts

@@ -0,0 +1,18 @@
+import type { DaemonResponse, SessionState } from '../types.ts';
+import type { RecordTraceDeps } from './record-trace-recording.ts';
+type AndroidDevice = SessionState['device'];
+type AndroidRecording = Extract<NonNullable<SessionState['recording']>, {
+    platform: 'android';
+}>;
+type AndroidRecordingBase = Pick<AndroidRecording, 'outPath' | 'clientOutPath' | 'telemetryPath' | 'startedAt' | 'showTouches' | 'gestureEvents'>;
+export declare function startAndroidRecording(params: {
+    deps: RecordTraceDeps;
+    device: AndroidDevice;
+    recordingBase: AndroidRecordingBase;
+}): Promise<DaemonResponse | AndroidRecording>;
+export declare function stopAndroidRecording(params: {
+    deps: RecordTraceDeps;
+    device: AndroidDevice;
+    recording: AndroidRecording;
+}): Promise<DaemonResponse | null>;
+export {};

package/dist/src/daemon/handlers/record-trace-ios.d.ts

@@ -0,0 +1,52 @@
+import { SessionStore } from '../session-store.ts';
+import type { DaemonRequest, DaemonResponse, SessionState } from '../types.ts';
+import type { RecordTraceDeps, RecordingBase } from './record-trace-recording.ts';
+export declare function normalizeAppBundleId(session: SessionState): string | undefined;
+export declare function warmIosSimulatorRunner(params: {
+    req: DaemonRequest;
+    activeSession: SessionState;
+    device: SessionState['device'];
+    logPath?: string;
+    deps: RecordTraceDeps;
+}): Promise<void>;
+export declare function startIosDeviceRecording(params: {
+    req: DaemonRequest;
+    activeSession: SessionState;
+    sessionStore: SessionStore;
+    device: SessionState['device'];
+    logPath?: string;
+    deps: RecordTraceDeps;
+    fpsFlag: number | undefined;
+    recordingBase: RecordingBase;
+    appBundleId: string;
+}): Promise<DaemonResponse | NonNullable<SessionState['recording']>>;
+export declare function startMacOsRecording(params: {
+    req: DaemonRequest;
+    activeSession: SessionState;
+    device: SessionState['device'];
+    logPath?: string;
+    deps: RecordTraceDeps;
+    fpsFlag: number | undefined;
+    recordingBase: RecordingBase;
+    appBundleId: string;
+}): Promise<DaemonResponse | NonNullable<SessionState['recording']>>;
+export declare function stopIosDeviceRecording(params: {
+    req: DaemonRequest;
+    activeSession: SessionState;
+    device: SessionState['device'];
+    logPath?: string;
+    deps: RecordTraceDeps;
+    recording: Extract<NonNullable<SessionState['recording']>, {
+        platform: 'ios-device-runner';
+    }>;
+}): Promise<DaemonResponse | null>;
+export declare function stopMacOsRecording(params: {
+    req: DaemonRequest;
+    activeSession: SessionState;
+    device: SessionState['device'];
+    logPath?: string;
+    deps: RecordTraceDeps;
+    recording: Extract<NonNullable<SessionState['recording']>, {
+        platform: 'macos-runner';
+    }>;
+}): Promise<DaemonResponse | null>;

package/dist/src/daemon/handlers/record-trace-recording.d.ts

@@ -0,0 +1,32 @@
+import { SessionStore } from '../session-store.ts';
+import type { DaemonRequest, DaemonResponse, RecordingGestureEvent } from '../types.ts';
+import { runCmd, runCmdBackground } from '../../utils/exec.ts';
+import { isPlayableVideo, waitForStableFile } from '../../utils/video.ts';
+import { writeRecordingTelemetry } from '../recording-telemetry.ts';
+import { runIosRunnerCommand } from '../../platforms/ios/runner-client.ts';
+import { overlayRecordingTouches, trimRecordingStart } from '../../recording/overlay.ts';
+export type RecordTraceDeps = {
+    runCmd: typeof runCmd;
+    runCmdBackground: typeof runCmdBackground;
+    runIosRunnerCommand: typeof runIosRunnerCommand;
+    waitForStableFile: typeof waitForStableFile;
+    isPlayableVideo: typeof isPlayableVideo;
+    writeRecordingTelemetry: typeof writeRecordingTelemetry;
+    trimRecordingStart: typeof trimRecordingStart;
+    overlayRecordingTouches: typeof overlayRecordingTouches;
+};
+export type RecordingBase = {
+    outPath: string;
+    clientOutPath?: string;
+    startedAt: number;
+    showTouches: boolean;
+    gestureEvents: RecordingGestureEvent[];
+};
+export declare function buildRecordTraceDeps(overrides?: Partial<RecordTraceDeps>): RecordTraceDeps;
+export declare function handleRecordCommand(params: {
+    req: DaemonRequest;
+    sessionName: string;
+    sessionStore: SessionStore;
+    logPath?: string;
+    deps?: Partial<RecordTraceDeps>;
+}): Promise<DaemonResponse>;

package/dist/src/daemon/handlers/record-trace.d.ts

@@ -1,15 +1,10 @@
-import { runCmd, runCmdBackground } from '../../utils/exec.ts';
-import { runIosRunnerCommand } from '../../platforms/ios/runner-client.ts';
 import type { DaemonRequest, DaemonResponse } from '../types.ts';
 import { SessionStore } from '../session-store.ts';
+import { type RecordTraceDeps } from './record-trace-recording.ts';
 export declare function handleRecordTraceCommands(params: {
     req: DaemonRequest;
     sessionName: string;
     sessionStore: SessionStore;
     logPath?: string;
-    deps?: {
-        runCmd: typeof runCmd;
-        runCmdBackground: typeof runCmdBackground;
-        runIosRunnerCommand: typeof runIosRunnerCommand;
-    };
+    deps?: Partial<RecordTraceDeps>;
 }): Promise<DaemonResponse | null>;

package/dist/src/daemon/handlers/snapshot-capture.d.ts

@@ -1,21 +1,28 @@
-import { dispatchCommand } from '../../core/dispatch.ts';
+import { dispatchCommand, type CommandFlags } from '../../core/dispatch.ts';
 import { type RawSnapshotNode, type SnapshotState } from '../../utils/snapshot.ts';
-import type { DaemonResponse, DaemonRequest, SessionState } from '../types.ts';
+import type { DaemonResponse, SessionState } from '../types.ts';
 type CaptureSnapshotParams = {
     dispatchSnapshotCommand: typeof dispatchCommand;
     device: SessionState['device'];
     session: SessionState | undefined;
-    req: DaemonRequest;
+    flags: CommandFlags | undefined;
+    outPath?: string;
     logPath: string;
     snapshotScope?: string;
 };
+type SnapshotData = {
+    nodes?: RawSnapshotNode[];
+    truncated?: boolean;
+    backend?: 'xctest' | 'android' | 'macos-helper';
+};
 export declare function captureSnapshot(params: CaptureSnapshotParams): Promise<{
     snapshot: SnapshotState;
 }>;
+export declare function captureSnapshotData(params: CaptureSnapshotParams): Promise<SnapshotData>;
 export declare function buildSnapshotState(data: {
     nodes?: RawSnapshotNode[];
     truncated?: boolean;
-    backend?: 'xctest' | 'android';
+    backend?: 'xctest' | 'android' | 'macos-helper';
 }, snapshotRaw: boolean | undefined): SnapshotState;
 export declare function resolveSnapshotScope(snapshotScope: string | undefined, session: SessionState | undefined): {
     ok: true;

package/dist/src/daemon/record-trace-errors.d.ts

@@ -0,0 +1,6 @@
+export declare function formatRecordTraceError(error: unknown): string;
+export declare function formatRecordTraceExecFailure(result: {
+    stdout: string;
+    stderr: string;
+    exitCode: number;
+}, command: string): string;

package/dist/src/daemon/recording-gestures.d.ts

@@ -0,0 +1,3 @@
+import type { SessionState } from './types.ts';
+export declare function recordTouchVisualizationEvent(session: SessionState, command: string, positionals: string[], result: Record<string, unknown> | void, fallback?: Record<string, unknown>, startedAtMs?: number, finishedAtMs?: number): void;
+export declare function augmentScrollVisualizationResult(session: SessionState, command: string, positionals: string[], result: Record<string, unknown> | void): Record<string, unknown> | void;

package/dist/src/daemon/recording-telemetry.d.ts

@@ -0,0 +1,20 @@
+import type { RecordingGestureEvent } from './types.ts';
+type RecordingTelemetryState = {
+    outPath: string;
+    gestureEvents: RecordingGestureEvent[];
+    telemetryPath?: string;
+};
+export declare function deriveRecordingTelemetryPath(videoPath: string): string;
+export declare function trimRecordingTelemetryEvents(events: RecordingGestureEvent[], trimStartMs: number): RecordingGestureEvent[];
+export declare function normalizeRecordingTelemetryEvents(events: RecordingGestureEvent[]): RecordingGestureEvent[];
+export declare function writeRecordingTelemetry(params: {
+    videoPath: string;
+    events: RecordingGestureEvent[];
+    trimStartMs?: number;
+}): string;
+export declare function persistRecordingTelemetry(params: {
+    recording: RecordingTelemetryState;
+    trimStartMs?: number;
+    writeTelemetry?: typeof writeRecordingTelemetry;
+}): string;
+export {};

package/dist/src/daemon/recording-timing.d.ts

@@ -0,0 +1,24 @@
+type GestureTimingSource = {
+    recordingStartedAt: number;
+    gestureClockOriginAtMs?: number;
+    gestureClockOriginUptimeMs?: number;
+    runnerStartedAtUptimeMs?: number;
+    gestureStartUptimeMs?: number;
+    gestureEndUptimeMs?: number;
+    fallbackStartedAtMs: number;
+    fallbackFinishedAtMs: number;
+};
+type GestureDurationSource = {
+    gestureStartUptimeMs?: number;
+    gestureEndUptimeMs?: number;
+    reportedDurationMs?: number;
+    fallbackStartedAtMs: number;
+    fallbackFinishedAtMs: number;
+};
+type TapVisualizationOffsetSource = GestureTimingSource & {
+    gestureDurationMs: number;
+};
+export declare function resolveGestureOffsetMs(source: GestureTimingSource): number;
+export declare function resolveGestureDurationMs(source: GestureDurationSource): number;
+export declare function resolveTapVisualizationOffsetMs(source: TapVisualizationOffsetSource): number;
+export {};

package/dist/src/daemon/request-router.d.ts

@@ -2,6 +2,8 @@ import { dispatchCommand } from '../core/dispatch.ts';
 import type { DaemonRequest, DaemonResponse } from './types.ts';
 import { SessionStore } from './session-store.ts';
 import type { LeaseRegistry } from './lease-registry.ts';
+import { snapshotAndroid, openAndroidApp, getAndroidAppState } from '../platforms/android/index.ts';
+import { runCmd } from '../utils/exec.ts';
 export type RequestRouterDeps = {
     logPath: string;
     token: string;
@@ -13,5 +15,9 @@ export type RequestRouterDeps = {
         fileName?: string;
     }) => string;
     dispatchCommand?: typeof dispatchCommand;
+    snapshotAndroidUi?: typeof snapshotAndroid;
+    reopenAndroidApp?: typeof openAndroidApp;
+    readAndroidAppState?: typeof getAndroidAppState;
+    execCommand?: typeof runCmd;
 };
 export declare function createRequestHandler(deps: RequestRouterDeps): (req: DaemonRequest) => Promise<DaemonResponse>;

package/dist/src/daemon/script-utils.d.ts

@@ -11,6 +11,7 @@ export declare function appendRuntimeHintFlags(parts: string[], flags: Pick<Sess
     bundleUrl?: string;
     launchUrl?: string;
 } | undefined): void;
+export declare function appendRecordActionScriptArgs(parts: string[], action: SessionAction): void;
 export declare function parseReplaySeriesFlags(command: string, args: string[]): {
     positionals: string[];
     flags: SessionAction['flags'];

package/dist/src/daemon/snapshot-processing.d.ts

@@ -7,3 +7,4 @@ export declare function normalizeType(type: string): string;
 export declare function isFillableType(type: string, platform: Platform): boolean;
 export declare function findNearestHittableAncestor(nodes: SnapshotState['nodes'], node: SnapshotState['nodes'][number]): SnapshotState['nodes'][number] | null;
 export declare function extractNodeText(node: SnapshotState['nodes'][number]): string;
+export declare function extractNodeReadText(node: SnapshotState['nodes'][number]): string;

package/dist/src/daemon/touch-reference-frame.d.ts

@@ -0,0 +1,7 @@
+import type { SnapshotNode, SnapshotState } from '../utils/snapshot.ts';
+export type TouchReferenceFrame = {
+    referenceWidth: number;
+    referenceHeight: number;
+};
+export declare function getSnapshotReferenceFrame(snapshot: SnapshotState | undefined): TouchReferenceFrame | undefined;
+export declare function inferTouchReferenceFrame(nodes: Array<Pick<SnapshotNode, 'type' | 'rect'>>): TouchReferenceFrame | undefined;

package/dist/src/daemon/types.d.ts

@@ -1,5 +1,6 @@
 import type { MaterializeInstallSource } from '../platforms/install-source.ts';
 import type { CommandFlags } from '../core/dispatch.ts';
+import type { SessionSurface } from '../core/session-surface.ts';
 import type { DeviceInfo, Platform, PlatformSelector } from '../utils/device.ts';
 import type { ExecResult } from '../utils/exec.ts';
 import type { SnapshotState } from '../utils/snapshot.ts';
@@ -63,10 +64,57 @@ export type DaemonResponse = {
         details?: Record<string, unknown>;
     };
 };
+type RecordingTelemetryBase = {
+    tMs: number;
+    x: number;
+    y: number;
+    referenceWidth?: number;
+    referenceHeight?: number;
+};
+type RecordingTelemetryTravel = RecordingTelemetryBase & {
+    x2: number;
+    y2: number;
+    durationMs: number;
+};
+export type RecordingGestureEvent = (RecordingTelemetryBase & {
+    kind: 'tap' | 'longpress';
+    durationMs?: number;
+}) | (RecordingTelemetryTravel & {
+    kind: 'swipe';
+}) | (RecordingTelemetryTravel & {
+    kind: 'scroll';
+    contentDirection: 'up' | 'down' | 'left' | 'right';
+    amount?: number;
+}) | (RecordingTelemetryTravel & {
+    kind: 'back-swipe';
+    edge: 'left' | 'right';
+}) | (RecordingTelemetryBase & {
+    kind: 'pinch';
+    scale: number;
+    durationMs: number;
+});
+type SessionRecordingBase = {
+    outPath: string;
+    clientOutPath?: string;
+    telemetryPath?: string;
+    overlayWarning?: string;
+    startedAt: number;
+    showTouches: boolean;
+    gestureEvents: RecordingGestureEvent[];
+    touchReferenceFrame?: {
+        referenceWidth: number;
+        referenceHeight: number;
+    };
+    gestureClockOriginAtMs?: number;
+    gestureClockOriginUptimeMs?: number;
+    runnerSessionId?: string;
+    invalidatedReason?: string;
+};
 export type SessionState = {
     name: string;
     device: DeviceInfo;
     createdAt: number;
+    surface?: SessionSurface;
     appBundleId?: string;
     appName?: string;
     snapshot?: SnapshotState;
@@ -77,23 +125,28 @@ export type SessionState = {
     recordSession?: boolean;
     saveScriptPath?: string;
     actions: SessionAction[];
-    recording?: {
-        platform: 'ios' | 'android';
-        outPath: string;
-        clientOutPath?: string;
-        remotePath?: string;
+    recording?: (SessionRecordingBase & {
+        platform: 'ios';
         child: ReturnType<typeof import('node:child_process').spawn>;
         wait: Promise<ExecResult>;
-    } | {
-        platform: 'ios-device-runner' | 'macos-runner';
-        outPath: string;
-        clientOutPath?: string;
         remotePath?: string;
-    };
+    }) | (SessionRecordingBase & {
+        platform: 'android';
+        remotePath: string;
+        remotePid: string;
+    }) | (SessionRecordingBase & {
+        platform: 'ios-device-runner';
+        remotePath: string;
+        runnerStartedAtUptimeMs?: number;
+        targetAppReadyUptimeMs?: number;
+    }) | (SessionRecordingBase & {
+        platform: 'macos-runner';
+        remotePath?: string;
+    });
     /** Session-scoped app log stream; logs written to outPath for agent to grep */
     appLog?: {
         platform: Platform;
-        backend: 'ios-simulator' | 'ios-device' | 'android';
+        backend: 'ios-simulator' | 'ios-device' | 'android' | 'macos';
         outPath: string;
         startedAt: number;
         getState: () => 'active' | 'failed';
@@ -117,3 +170,4 @@ export type SessionAction = {
     };
     result?: Record<string, unknown>;
 };
+export {};