@midscene/android 1.12.0 → 1.12.1-beta-20260824081858.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/es/index.mjs CHANGED
@@ -8,7 +8,7 @@ import { getMidsceneLocationSchema, z } from "@midscene/core";
8
8
  import { createDefaultMobileActions, defineAction } from "@midscene/core/device";
9
9
  import { getTmpFile, sleep } from "@midscene/core/utils";
10
10
  import { MIDSCENE_ADB_PATH, MIDSCENE_ADB_REMOTE_HOST, MIDSCENE_ADB_REMOTE_PORT, MIDSCENE_ANDROID_IME_STRATEGY, MIDSCENE_ANDROID_SCREENSHOT_STRATEGY, globalConfigManager, overrideAIConfig } from "@midscene/shared/env";
11
- import { createImgBase64ByFormat, validateScreenshotBuffer } from "@midscene/shared/img";
11
+ import { createImgBase64ByFormat, imageInfoOfBase64, resizeBase64ImageToJpeg, validateScreenshotBuffer } from "@midscene/shared/img";
12
12
  import { mergeAndNormalizeAppNameMapping, normalizeForComparison, repeat } from "@midscene/shared/utils";
13
13
  import { ADB, getSdkRootFromEnv } from "appium-adb";
14
14
  import { Agent } from "@midscene/core/agent";
@@ -29,7 +29,6 @@ var __webpack_modules__ = {
29
29
  __webpack_require__.d(__webpack_exports__, {
30
30
  GJ: ()=>isScrcpyFreshFrameUnavailableError,
31
31
  ScrcpyScreenshotManager: ()=>ScrcpyScreenshotManager,
32
- Xr: ()=>SCRCPY_VIDEO_BIT_RATE_NETWORK_HINT,
33
32
  ov: ()=>DEFAULT_SCRCPY_CONFIG
34
33
  });
35
34
  var node_fs__rspack_import_0 = __webpack_require__("node:fs");
@@ -63,7 +62,6 @@ var __webpack_modules__ = {
63
62
  const FRESH_FRAME_TIMEOUT_MS = 300;
64
63
  const KEYFRAME_POLL_INTERVAL_MS = 200;
65
64
  const MAX_SCAN_BYTES = 1000;
66
- const CONNECTION_WAIT_MS = 1000;
67
65
  const MAX_SERVER_OUTPUT_LINES = 100;
68
66
  const SERVER_OUTPUT_DRAIN_TIMEOUT_MS = 500;
69
67
  const MAX_FRAME_AGE_US = 500000n;
@@ -73,7 +71,12 @@ var __webpack_modules__ = {
73
71
  'dumpsys',
74
72
  'power'
75
73
  ];
76
- const SCRCPY_VIDEO_BIT_RATE_NETWORK_HINT = 'The appropriate scrcpy video bitrate depends on network conditions. For constrained remote links, consider setting scrcpyConfig.videoBitRate to 4_000_000 (4 Mbps) as a starting point, and lower it further if backlog persists.';
74
+ const SCRCPY_VIDEO_BIT_RATE_NETWORK_HINT = 'The appropriate scrcpy video bitrate depends on network conditions. For constrained remote links, pass --scrcpy-video-bit-rate 4000000 in the Android CLI, or set scrcpyConfig.videoBitRate to 4_000_000 (4 Mbps) in SDK/YAML configuration. Lower it further if backlog persists.';
75
+ const CONSTRAINED_LINK_STARTING_VIDEO_BIT_RATE = 4000000;
76
+ function getScrcpyVideoBitRateNetworkHint(currentVideoBitRate) {
77
+ if (currentVideoBitRate <= CONSTRAINED_LINK_STARTING_VIDEO_BIT_RATE) return 'The current scrcpy video bitrate is already at or below 4 Mbps. Lowering it further is unlikely to help on a local USB connection; check whether the screen was static or the device encoder emitted no new frame.';
78
+ return SCRCPY_VIDEO_BIT_RATE_NETWORK_HINT;
79
+ }
77
80
  const BUSY_LOOP_WINDOW_MS = 1000;
78
81
  const BUSY_LOOP_MAX_READS = 500;
79
82
  const BUSY_LOOP_COOLDOWN_MS = 50;
@@ -87,8 +90,9 @@ var __webpack_modules__ = {
87
90
  const SCRCPY_FRESH_FRAME_UNAVAILABLE_ERROR_CODE = 'ERR_SCRCPY_FRESH_FRAME_UNAVAILABLE';
88
91
  class ScrcpyFreshFrameUnavailableError extends Error {
89
92
  constructor(message, options){
90
- super(message, options), _define_property(this, "code", SCRCPY_FRESH_FRAME_UNAVAILABLE_ERROR_CODE);
93
+ super(message, options), _define_property(this, "code", SCRCPY_FRESH_FRAME_UNAVAILABLE_ERROR_CODE), _define_property(this, "diagnosticMessage", void 0);
91
94
  this.name = 'ScrcpyFreshFrameUnavailableError';
95
+ this.diagnosticMessage = options?.diagnosticMessage;
92
96
  }
93
97
  }
94
98
  function isScrcpyFreshFrameUnavailableError(error) {
@@ -121,26 +125,30 @@ var __webpack_modules__ = {
121
125
  await this.ensureFfmpegAvailable();
122
126
  }
123
127
  async ensureConnected() {
128
+ if (this.disposed) throw new Error('Scrcpy manager has been disposed');
124
129
  if (this.scrcpyClient && this.videoStream) {
125
130
  debugScrcpy('Scrcpy already connected');
126
131
  await this.ensureFrameClockCalibration();
127
132
  this.resetIdleTimer();
128
133
  return;
129
134
  }
130
- if (this.isConnecting) {
131
- debugScrcpy('Connection already in progress, waiting...');
132
- await new Promise((resolve)=>setTimeout(resolve, CONNECTION_WAIT_MS));
133
- if (this.scrcpyClient && this.videoStream) {
134
- await this.ensureFrameClockCalibration();
135
- this.resetIdleTimer();
136
- return;
137
- }
138
- throw new Error('Scrcpy connection failed: another connection attempt did not complete in time');
135
+ if (this.connectionPromise) {
136
+ debugScrcpy('Connection already in progress, sharing the same attempt...');
137
+ await this.connectionPromise;
138
+ return;
139
139
  }
140
+ const connectionPromise = this.connectScrcpy();
141
+ this.connectionPromise = connectionPromise;
142
+ try {
143
+ await connectionPromise;
144
+ } finally{
145
+ if (this.connectionPromise === connectionPromise) this.connectionPromise = null;
146
+ }
147
+ }
148
+ async connectScrcpy() {
140
149
  const serverOutput = [];
141
150
  let serverOutputTask = null;
142
151
  try {
143
- this.isConnecting = true;
144
152
  debugScrcpy('Starting scrcpy connection...');
145
153
  const { AdbScrcpyClient } = await import("@yume-chan/adb-scrcpy");
146
154
  const { ReadableStream } = await import("@yume-chan/stream-extra");
@@ -160,6 +168,8 @@ var __webpack_modules__ = {
160
168
  height
161
169
  };
162
170
  await this.ensureFrameClockCalibration();
171
+ this.streamBaselineFramePending = true;
172
+ this.streamBaselineFrameDeadlineAt = Date.now() + MAX_KEYFRAME_WAIT_MS;
163
173
  this.startFrameConsumer();
164
174
  this.resetIdleTimer();
165
175
  this.isInitialized = true;
@@ -172,8 +182,6 @@ var __webpack_modules__ = {
172
182
  new Promise((resolve)=>setTimeout(resolve, SERVER_OUTPUT_DRAIN_TIMEOUT_MS))
173
183
  ]);
174
184
  throw this.createConnectionError(error, serverOutput);
175
- } finally{
176
- this.isConnecting = false;
177
185
  }
178
186
  }
179
187
  async createScrcpyOptions() {
@@ -358,24 +366,30 @@ var __webpack_modules__ = {
358
366
  if (this.deviceClockCalibrationPromise === calibrationPromise) this.deviceClockCalibrationPromise = null;
359
367
  }
360
368
  }
361
- async setFreshnessBarrier(reason) {
369
+ async setFreshnessBarrier(reason, options = {}) {
370
+ const cachedFrame = this.getCachedKeyframeCandidate();
362
371
  const generation = ++this.frameFreshnessBarrierGeneration;
363
372
  this.frameFreshnessBarrierPending = true;
373
+ this.frameFreshnessBarrierAllowsOverAgeForNextCapture = false;
364
374
  this.clearFrameCache();
365
375
  try {
366
376
  const calibration = this.deviceClockCalibration;
367
377
  if (!calibration) throw new Error('Scrcpy frame clock is not calibrated for the current stream epoch');
368
- const hostMonotonicUs = this.monotonicTimeUs();
378
+ const hostMonotonicUs = options.hostMonotonicUs ?? this.monotonicTimeUs();
369
379
  const estimatedDeviceNowUs = this.estimateDeviceTimeUs(hostMonotonicUs, calibration);
370
380
  const conservativeDeviceNowUs = estimatedDeviceNowUs + this.getCalibrationUncertaintyUs(calibration);
371
381
  const barrierPtsUs = (conservativeDeviceNowUs / 1000n + 1n) * 1000n;
372
382
  if (generation !== this.frameFreshnessBarrierGeneration) return this.frameFreshnessBarrierPtsUs ?? barrierPtsUs;
373
383
  this.frameFreshnessBarrierPtsUs = null === this.frameFreshnessBarrierPtsUs || barrierPtsUs > this.frameFreshnessBarrierPtsUs ? barrierPtsUs : this.frameFreshnessBarrierPtsUs;
374
384
  this.frameFreshnessBarrierReason = reason;
385
+ this.frameFreshnessBarrierAllowsOverAgeForNextCapture = options.allowOverAgeForNextCapture ?? false;
375
386
  this.frameFreshnessBarrierPending = false;
376
387
  this.frameFreshnessError = null;
377
388
  this.lastFramePtsUs = null;
378
- this.clearFrameCache();
389
+ if (cachedFrame?.ptsUs !== void 0 && cachedFrame.ptsUs >= this.frameFreshnessBarrierPtsUs) {
390
+ this.restoreFrameCache(cachedFrame);
391
+ debugScrcpy(`Preserved cached frame PTS ${cachedFrame.ptsUs}µs because it crosses the newly armed ${reason} barrier`);
392
+ } else this.clearFrameCache();
379
393
  debugScrcpy(`Armed frame freshness barrier at PTS ${this.frameFreshnessBarrierPtsUs}µs (${reason}, projected from stream-epoch clock anchor, uncertainty<=${Number(this.getCalibrationUncertaintyUs(calibration)) / 1000}ms)`);
380
394
  return this.frameFreshnessBarrierPtsUs;
381
395
  } catch (error) {
@@ -396,6 +410,7 @@ var __webpack_modules__ = {
396
410
  this.frameFreshnessBarrierPending = true;
397
411
  this.frameFreshnessBarrierPtsUs = null;
398
412
  this.frameFreshnessBarrierReason = null;
413
+ this.frameFreshnessBarrierAllowsOverAgeForNextCapture = false;
399
414
  this.deviceClockCalibration = null;
400
415
  this.clearFrameCache();
401
416
  this.warnFrameFreshness();
@@ -417,7 +432,7 @@ var __webpack_modules__ = {
417
432
  const behindBarrierUs = this.frameFreshnessBarrierPtsUs - packetPtsUs;
418
433
  this.frameFreshnessError = new Error(`Scrcpy frame predates the ${this.frameFreshnessBarrierReason ?? 'active'} freshness barrier by ${Number(behindBarrierUs) / 1000}ms; refusing to use it`);
419
434
  this.clearFrameCache();
420
- this.warnFrameFreshness();
435
+ debugScrcpy(this.frameFreshnessError.message);
421
436
  return false;
422
437
  }
423
438
  estimateFrameAgeUs(packetPtsUs, hostMonotonicUs = this.monotonicTimeUs()) {
@@ -467,13 +482,18 @@ var __webpack_modules__ = {
467
482
  this.lastFrameFreshnessWarningAt = now;
468
483
  }
469
484
  }
470
- warnTransportBacklog(error) {
471
- const now = Date.now();
472
- if (now - this.lastTransportBacklogWarningAt < TRANSPORT_BACKLOG_WARN_INTERVAL_MS) return;
485
+ transportBacklogMessage(error) {
473
486
  const cause = this.frameFreshnessError ?? error;
474
487
  const causeMessage = cause instanceof Error ? cause.message : String(cause);
488
+ if (this.streamBaselineFramePending && null === this.frameFreshnessBarrierPtsUs) return `The new scrcpy stream did not produce a usable baseline frame within its ${MAX_KEYFRAME_WAIT_MS}ms startup window; closing the stream epoch and falling back to ADB screenshot. This is a stream startup or encoder-readiness failure, not evidence that the configured video bitrate is too high.\nError: ${causeMessage}`;
475
489
  const currentBitRateMbps = this.options.videoBitRate / 1000000;
476
- warnScrcpy(`No usable scrcpy frame crossed the active freshness target within ${FRESH_FRAME_TIMEOUT_MS}ms; closing the stale stream epoch and falling back to ADB screenshot. This may indicate transport backlog or a static screen that emitted no new frame. ${SCRCPY_VIDEO_BIT_RATE_NETWORK_HINT} Current videoBitRate: ${this.options.videoBitRate} bps (${currentBitRateMbps} Mbps).\nError: ${causeMessage}`);
490
+ const networkHint = getScrcpyVideoBitRateNetworkHint(this.options.videoBitRate);
491
+ return `No usable scrcpy frame crossed the active freshness target within ${FRESH_FRAME_TIMEOUT_MS}ms; closing the stale stream epoch and falling back to ADB screenshot. This may indicate transport backlog or a static screen that emitted no new frame. ${networkHint} Current videoBitRate: ${this.options.videoBitRate} bps (${currentBitRateMbps} Mbps).\nError: ${causeMessage}`;
492
+ }
493
+ warnTransportBacklog(error) {
494
+ const now = Date.now();
495
+ if (now - this.lastTransportBacklogWarningAt < TRANSPORT_BACKLOG_WARN_INTERVAL_MS) return;
496
+ warnScrcpy(this.transportBacklogMessage(error));
477
497
  this.lastTransportBacklogWarningAt = now;
478
498
  }
479
499
  estimateFrameTiming(packetPtsUs, receivedAtUs) {
@@ -495,12 +515,21 @@ var __webpack_modules__ = {
495
515
  this.lastRawKeyframePtsUs = void 0;
496
516
  this.lastRawKeyframeEstimatedAgeMs = void 0;
497
517
  }
518
+ restoreFrameCache(frame) {
519
+ this.lastRawKeyframe = frame.data;
520
+ this.lastRawKeyframeAt = frame.capturedAt;
521
+ this.lastRawKeyframePtsUs = frame.ptsUs;
522
+ this.lastRawKeyframeEstimatedAgeMs = frame.estimatedAgeMs;
523
+ }
498
524
  monotonicTimeUs() {
499
525
  return process.hrtime.bigint() / 1000n;
500
526
  }
501
527
  resetFrameFreshnessState() {
502
528
  this.frameFreshnessBarrierPtsUs = null;
503
529
  this.frameFreshnessBarrierReason = null;
530
+ this.frameFreshnessBarrierAllowsOverAgeForNextCapture = false;
531
+ this.streamBaselineFramePending = false;
532
+ this.streamBaselineFrameDeadlineAt = 0;
504
533
  this.frameFreshnessBarrierPending = false;
505
534
  this.frameFreshnessBarrierGeneration = 0;
506
535
  this.deviceClockCalibration = null;
@@ -532,35 +561,52 @@ var __webpack_modules__ = {
532
561
  capturedAt: this.lastRawKeyframeAt
533
562
  };
534
563
  }
564
+ canReuseFrameAcrossActionWait(frame) {
565
+ return this.frameFreshnessBarrierAllowsOverAgeForNextCapture && null !== this.frameFreshnessBarrierPtsUs && void 0 !== frame.ptsUs && frame.ptsUs >= this.frameFreshnessBarrierPtsUs;
566
+ }
567
+ canUseStreamBaseline(frame) {
568
+ return this.streamBaselineFramePending && !this.frameFreshnessBarrierPending && null === this.frameFreshnessBarrierPtsUs && Date.now() <= this.streamBaselineFrameDeadlineAt && void 0 !== frame.ptsUs;
569
+ }
570
+ consumeActionFreshnessBarrier(frame) {
571
+ if (!this.canReuseFrameAcrossActionWait(frame)) return;
572
+ this.frameFreshnessBarrierPtsUs = null;
573
+ this.frameFreshnessBarrierReason = null;
574
+ this.frameFreshnessBarrierAllowsOverAgeForNextCapture = false;
575
+ this.frameFreshnessError = null;
576
+ }
535
577
  async decodeRawKeyframeToJpeg(frame) {
536
578
  return this.decodeH264ToJpeg(Buffer.concat([
537
579
  frame.header,
538
580
  frame.data
539
581
  ]));
540
582
  }
541
- async waitForUsableKeyframe(timeoutMs) {
542
- const deadline = Date.now() + timeoutMs;
543
- let candidate = this.getCachedKeyframeCandidate();
544
- while(true){
545
- if (candidate && this.isFrameAgeAcceptable(candidate.ptsUs)) return candidate;
546
- const remainingMs = deadline - Date.now();
547
- if (remainingMs <= 0) throw new Error(`No fresh keyframe received within ${timeoutMs}ms`);
548
- candidate = await this.waitForNextKeyframe(remainingMs);
549
- }
550
- }
551
583
  async prepareFreshFrame() {
552
584
  await this.ensureConnected();
553
585
  await this.ensureFrameClockCalibration();
554
586
  await this.waitForKeyframe();
555
- await this.waitForUsableKeyframe(MAX_KEYFRAME_WAIT_MS);
587
+ await this.waitForPlanningFrame();
556
588
  }
557
589
  async waitForPlanningFrame() {
558
590
  let planningBarrierArmed = false;
559
- let deadline = Date.now() + FRESH_FRAME_TIMEOUT_MS;
591
+ const planningStartedAt = Date.now();
592
+ const startupWindowRemainingMs = this.streamBaselineFramePending ? this.streamBaselineFrameDeadlineAt - planningStartedAt : 0;
593
+ if (this.streamBaselineFramePending && startupWindowRemainingMs <= 0) {
594
+ this.streamBaselineFramePending = false;
595
+ this.streamBaselineFrameDeadlineAt = 0;
596
+ }
597
+ let deadline = planningStartedAt + (startupWindowRemainingMs > 0 ? startupWindowRemainingMs : FRESH_FRAME_TIMEOUT_MS);
560
598
  let candidate = this.getCachedKeyframeCandidate();
561
599
  while(true){
562
600
  if (candidate) {
563
601
  if (void 0 === candidate.ptsUs) throw new Error('Scrcpy frame has no PTS metadata; cannot prove planning freshness');
602
+ if (this.canReuseFrameAcrossActionWait(candidate)) {
603
+ debugScrcpy(`Using frame PTS ${candidate.ptsUs}µs that crossed the active input-action barrier; preserving the settled frame across wait-after-action`);
604
+ return candidate;
605
+ }
606
+ if (this.canUseStreamBaseline(candidate)) {
607
+ debugScrcpy(`Using frame PTS ${candidate.ptsUs}µs as the first planning baseline for the new scrcpy stream epoch`);
608
+ return candidate;
609
+ }
564
610
  const age = this.estimateFrameAge(candidate.ptsUs);
565
611
  if (null === age) throw new Error('Scrcpy frame clock is not calibrated; cannot prove planning freshness');
566
612
  if (age.upperBoundUs <= MAX_FRAME_AGE_US) return candidate;
@@ -578,11 +624,12 @@ var __webpack_modules__ = {
578
624
  }
579
625
  }
580
626
  async closeStaleStreamAndCreateFallbackError(error) {
581
- this.warnTransportBacklog(error);
627
+ const diagnosticMessage = this.transportBacklogMessage(error);
582
628
  const causeMessage = error instanceof Error ? error.message : String(error);
583
629
  await this.disconnect();
584
- return new ScrcpyFreshFrameUnavailableError(`Unable to obtain a fresh scrcpy frame; the stale stream epoch was closed so the caller can use ADB screenshot fallback. ${causeMessage}`, {
585
- cause: error
630
+ return new ScrcpyFreshFrameUnavailableError(`Unable to obtain a fresh scrcpy frame; the stale stream epoch was closed so the caller can restart it or use ADB screenshot fallback. ${causeMessage}`, {
631
+ cause: error,
632
+ diagnosticMessage
586
633
  });
587
634
  }
588
635
  async getScreenshotJpeg() {
@@ -609,6 +656,9 @@ var __webpack_modules__ = {
609
656
  const t5 = Date.now();
610
657
  const result = await this.decodeH264ToJpeg(keyframeBuffer);
611
658
  const decodeTime = Date.now() - t5;
659
+ this.consumeActionFreshnessBarrier(frame);
660
+ this.streamBaselineFramePending = false;
661
+ this.streamBaselineFrameDeadlineAt = 0;
612
662
  const totalTime = Date.now() - perfStart;
613
663
  debugScrcpy(`Performance: total=${totalTime}ms (connect=${connectTime}ms, frameWait=${frameWaitTime}ms, decode=${decodeTime}ms)`);
614
664
  return result;
@@ -765,6 +815,13 @@ var __webpack_modules__ = {
765
815
  }
766
816
  debugScrcpy('Scrcpy disconnected');
767
817
  }
818
+ async dispose() {
819
+ if (this.disposed) return;
820
+ this.disposed = true;
821
+ if (this.connectionPromise) await this.connectionPromise.catch(()=>{});
822
+ await this.disconnect();
823
+ await this.adb.close();
824
+ }
768
825
  isConnected() {
769
826
  return this.isInitialized && null !== this.scrcpyClient;
770
827
  }
@@ -774,7 +831,8 @@ var __webpack_modules__ = {
774
831
  _define_property(this, "videoStream", null);
775
832
  _define_property(this, "spsHeader", null);
776
833
  _define_property(this, "idleTimer", null);
777
- _define_property(this, "isConnecting", false);
834
+ _define_property(this, "connectionPromise", null);
835
+ _define_property(this, "disposed", false);
778
836
  _define_property(this, "isInitialized", false);
779
837
  _define_property(this, "options", void 0);
780
838
  _define_property(this, "ffmpegAvailable", null);
@@ -788,6 +846,9 @@ var __webpack_modules__ = {
788
846
  _define_property(this, "streamReader", null);
789
847
  _define_property(this, "frameFreshnessBarrierPtsUs", null);
790
848
  _define_property(this, "frameFreshnessBarrierReason", null);
849
+ _define_property(this, "frameFreshnessBarrierAllowsOverAgeForNextCapture", false);
850
+ _define_property(this, "streamBaselineFramePending", false);
851
+ _define_property(this, "streamBaselineFrameDeadlineAt", 0);
791
852
  _define_property(this, "frameFreshnessBarrierPending", false);
792
853
  _define_property(this, "frameFreshnessBarrierGeneration", 0);
793
854
  _define_property(this, "deviceClockCalibration", null);
@@ -927,6 +988,9 @@ function _define_property(obj, key, value) {
927
988
  return obj;
928
989
  }
929
990
  const debugAdapter = (0, logger_.getDebug)('android:scrcpy-adapter');
991
+ const warnAdapter = (0, logger_.getDebug)('android:scrcpy-adapter', {
992
+ console: true
993
+ });
930
994
  const SCRCPY_RETRY_COOLDOWN_MS = 5000;
931
995
  const DEFAULT_ADB_SERVER_ENDPOINT = {
932
996
  host: '127.0.0.1',
@@ -980,10 +1044,11 @@ class ScrcpyDeviceAdapter {
980
1044
  if (null === this.retryAfter || Date.now() >= this.retryAfter) return;
981
1045
  throw new Error(`scrcpy retry is cooling down until ${new Date(this.retryAfter).toISOString()}. Last error: ${this.lastError}`);
982
1046
  }
983
- resolveConfig(deviceInfo) {
1047
+ resolveConfig(_deviceInfo) {
984
1048
  if (this.resolvedConfig) return this.resolvedConfig;
985
1049
  const config = this.scrcpyConfig;
986
1050
  const maxSize = config?.maxSize ?? scrcpy_manager.ov.maxSize;
1051
+ if (!Number.isInteger(maxSize) || maxSize < 0) throw new Error(`Invalid scrcpyConfig.maxSize: expected a non-negative integer, received ${maxSize}`);
987
1052
  const videoBitRate = config?.videoBitRate ?? scrcpy_manager.ov.videoBitRate;
988
1053
  this.resolvedConfig = {
989
1054
  enabled: this.isConfigured(),
@@ -993,31 +1058,68 @@ class ScrcpyDeviceAdapter {
993
1058
  };
994
1059
  return this.resolvedConfig;
995
1060
  }
1061
+ async prepareFallbackScreenshot(screenshotBase64) {
1062
+ if (!this.isConfigured()) return screenshotBase64;
1063
+ const { maxSize } = this.resolveConfig();
1064
+ if (0 === maxSize) return screenshotBase64;
1065
+ const sourceSize = await imageInfoOfBase64(screenshotBase64);
1066
+ const largestDimension = Math.max(sourceSize.width, sourceSize.height);
1067
+ if (largestDimension <= maxSize) return screenshotBase64;
1068
+ const scale = maxSize / largestDimension;
1069
+ const targetSize = {
1070
+ width: Math.max(1, Math.round(sourceSize.width * scale)),
1071
+ height: Math.max(1, Math.round(sourceSize.height * scale))
1072
+ };
1073
+ debugAdapter(`Applying scrcpy maxSize to fallback screenshot: ${sourceSize.width}x${sourceSize.height} -> ${targetSize.width}x${targetSize.height}`);
1074
+ return resizeBase64ImageToJpeg(screenshotBase64, {
1075
+ sourceSize,
1076
+ targetSize
1077
+ });
1078
+ }
996
1079
  async ensureManager(deviceInfo) {
997
1080
  if (this.manager) return this.manager;
1081
+ if (this.managerPromise) return this.managerPromise;
998
1082
  debugAdapter('Initializing Scrcpy manager...');
1083
+ const generation = this.lifecycleGeneration;
1084
+ const managerPromise = (async ()=>{
1085
+ let adb = null;
1086
+ let manager = null;
1087
+ try {
1088
+ const { Adb, AdbServerClient } = await import("@yume-chan/adb");
1089
+ const { AdbServerNodeTcpConnector } = await import("@yume-chan/adb-server-node-tcp");
1090
+ const { ScrcpyScreenshotManager: ScrcpyManager } = await Promise.resolve().then(__webpack_require__.bind(__webpack_require__, "./src/scrcpy-manager.ts"));
1091
+ const adbServerEndpoint = await this.resolveAdbServerEndpoint();
1092
+ const adbClient = new AdbServerClient(new AdbServerNodeTcpConnector(adbServerEndpoint));
1093
+ adb = new Adb(await adbClient.createTransport({
1094
+ serial: this.deviceId
1095
+ }));
1096
+ const config = this.resolveConfig(deviceInfo);
1097
+ manager = new ScrcpyManager(adb, {
1098
+ maxSize: config.maxSize,
1099
+ videoBitRate: config.videoBitRate,
1100
+ idleTimeoutMs: config.idleTimeoutMs
1101
+ });
1102
+ await manager.validateEnvironment();
1103
+ if (generation !== this.lifecycleGeneration) throw new Error('Scrcpy manager initialization was superseded by device cleanup');
1104
+ this.manager = manager;
1105
+ debugAdapter('Scrcpy manager initialized');
1106
+ return manager;
1107
+ } catch (error) {
1108
+ try {
1109
+ if (manager) await manager.dispose();
1110
+ else await adb?.close();
1111
+ } catch (cleanupError) {
1112
+ debugAdapter(`Failed to clean up Scrcpy manager initialization: ${cleanupError}`);
1113
+ }
1114
+ debugAdapter(`Failed to initialize Scrcpy manager: ${error}`);
1115
+ throw new Error(`Failed to initialize Scrcpy for device ${this.deviceId}. Ensure ADB server is running and device is connected. Error: ${error}`);
1116
+ }
1117
+ })();
1118
+ this.managerPromise = managerPromise;
999
1119
  try {
1000
- const { Adb, AdbServerClient } = await import("@yume-chan/adb");
1001
- const { AdbServerNodeTcpConnector } = await import("@yume-chan/adb-server-node-tcp");
1002
- const { ScrcpyScreenshotManager: ScrcpyManager } = await Promise.resolve().then(__webpack_require__.bind(__webpack_require__, "./src/scrcpy-manager.ts"));
1003
- const adbServerEndpoint = await this.resolveAdbServerEndpoint();
1004
- const adbClient = new AdbServerClient(new AdbServerNodeTcpConnector(adbServerEndpoint));
1005
- const adb = new Adb(await adbClient.createTransport({
1006
- serial: this.deviceId
1007
- }));
1008
- const config = this.resolveConfig(deviceInfo);
1009
- const manager = new ScrcpyManager(adb, {
1010
- maxSize: config.maxSize,
1011
- videoBitRate: config.videoBitRate,
1012
- idleTimeoutMs: config.idleTimeoutMs
1013
- });
1014
- await manager.validateEnvironment();
1015
- this.manager = manager;
1016
- debugAdapter('Scrcpy manager initialized');
1017
- return this.manager;
1018
- } catch (error) {
1019
- debugAdapter(`Failed to initialize Scrcpy manager: ${error}`);
1020
- throw new Error(`Failed to initialize Scrcpy for device ${this.deviceId}. Ensure ADB server is running and device is connected. Error: ${error}`);
1120
+ return await managerPromise;
1121
+ } finally{
1122
+ if (this.managerPromise === managerPromise) this.managerPromise = null;
1021
1123
  }
1022
1124
  }
1023
1125
  async screenshotBase64(deviceInfo) {
@@ -1029,16 +1131,54 @@ class ScrcpyDeviceAdapter {
1029
1131
  await this.applyPendingActionBarrier(manager);
1030
1132
  const screenshotBuffer = await manager.getScreenshotJpeg();
1031
1133
  this.clearFailure();
1032
- return createImgBase64ByFormat('jpeg', screenshotBuffer.toString('base64'));
1134
+ return this.jpegBufferToBase64(screenshotBuffer);
1033
1135
  } catch (error) {
1034
- if ((0, scrcpy_manager.GJ)(error)) {
1035
- this.markFreshnessRecoveryPending(manager, error);
1136
+ if (!(0, scrcpy_manager.GJ)(error)) {
1137
+ this.recordFailure(error);
1036
1138
  throw error;
1037
1139
  }
1038
- this.recordFailure(error);
1039
- throw error;
1140
+ this.markFreshnessRecoveryPending(error);
1141
+ debugAdapter(`Scrcpy freshness target was unavailable; restarting the stream once before ADB fallback: ${error}`);
1142
+ try {
1143
+ const screenshotBuffer = await this.restartAndCaptureOnce(deviceInfo);
1144
+ manager = this.manager;
1145
+ if (!manager) throw new Error('Scrcpy manager disappeared after freshness recovery');
1146
+ this.attachKeyframeListeners(manager);
1147
+ this.clearFailure();
1148
+ debugAdapter('Scrcpy screenshot recovered on a new stream epoch');
1149
+ return this.jpegBufferToBase64(screenshotBuffer);
1150
+ } catch (retryError) {
1151
+ if ((0, scrcpy_manager.GJ)(retryError)) this.markFreshnessRecoveryPending(retryError);
1152
+ else this.recordFailure(retryError);
1153
+ this.warnFreshnessFallback(error, retryError);
1154
+ throw retryError;
1155
+ }
1040
1156
  }
1041
1157
  }
1158
+ async restartAndCaptureOnce(deviceInfo) {
1159
+ if (this.freshnessRestartPromise) return this.freshnessRestartPromise;
1160
+ const generation = this.lifecycleGeneration;
1161
+ const restartPromise = (async ()=>{
1162
+ const manager = await this.ensureManager(deviceInfo);
1163
+ await manager.ensureConnected();
1164
+ await this.applyPendingActionBarrier(manager);
1165
+ if (generation !== this.lifecycleGeneration) throw new Error('Scrcpy freshness restart was cancelled by cleanup');
1166
+ return manager.getScreenshotJpeg();
1167
+ })();
1168
+ this.freshnessRestartPromise = restartPromise;
1169
+ try {
1170
+ return await restartPromise;
1171
+ } finally{
1172
+ if (this.freshnessRestartPromise === restartPromise) this.freshnessRestartPromise = null;
1173
+ }
1174
+ }
1175
+ jpegBufferToBase64(screenshotBuffer) {
1176
+ return createImgBase64ByFormat('jpeg', screenshotBuffer.toString('base64'));
1177
+ }
1178
+ warnFreshnessFallback(firstError, retryError) {
1179
+ const retryDiagnostic = (0, scrcpy_manager.GJ)(retryError) ? retryError.diagnosticMessage : void 0;
1180
+ warnAdapter(retryDiagnostic ?? (firstError.diagnosticMessage ? `${firstError.diagnosticMessage}\nScrcpy stream restart error: ${retryError}` : void 0) ?? `Scrcpy stream restart failed; falling back to ADB screenshot. Error: ${retryError}`);
1181
+ }
1042
1182
  async subscribeKeyframes(deviceInfo, listener) {
1043
1183
  this.ensureRetryReady();
1044
1184
  this.keyframeListeners.add(listener);
@@ -1071,17 +1211,26 @@ class ScrcpyDeviceAdapter {
1071
1211
  this.keyframeUnsubscribers.clear();
1072
1212
  for (const listener of this.keyframeListeners)this.attachKeyframeListener(manager, listener);
1073
1213
  }
1074
- markFreshnessRecoveryPending(manager, error) {
1214
+ markFreshnessRecoveryPending(error) {
1075
1215
  this.lastError = error.message;
1076
1216
  this.retryAfter = null;
1077
1217
  this.freshnessRecoveryPending = true;
1078
1218
  this.keyframeUnsubscribers.clear();
1079
- if (manager && this.manager === manager) this.manager = null;
1080
1219
  }
1081
1220
  async applyPendingActionBarrier(manager) {
1082
- if (!this.pendingActionBarrier) return;
1083
- await manager.setFreshnessBarrier('completed input action while scrcpy was unavailable');
1084
- this.pendingActionBarrier = false;
1221
+ const actionCompletedAtHostUs = this.pendingActionBarrierAtHostUs;
1222
+ if (null === actionCompletedAtHostUs) return;
1223
+ await manager.setFreshnessBarrier('completed input action while scrcpy was unavailable', {
1224
+ allowOverAgeForNextCapture: true,
1225
+ hostMonotonicUs: actionCompletedAtHostUs
1226
+ });
1227
+ if (this.pendingActionBarrierAtHostUs === actionCompletedAtHostUs) this.pendingActionBarrierAtHostUs = null;
1228
+ }
1229
+ monotonicTimeUs() {
1230
+ return process.hrtime.bigint() / 1000n;
1231
+ }
1232
+ deferActionBarrier(actionCompletedAtHostUs) {
1233
+ if (null === this.pendingActionBarrierAtHostUs || actionCompletedAtHostUs > this.pendingActionBarrierAtHostUs) this.pendingActionBarrierAtHostUs = actionCompletedAtHostUs;
1085
1234
  }
1086
1235
  recoverAfterAdbScreenshot(deviceInfo) {
1087
1236
  if (!this.freshnessRecoveryPending || this.recoveryPromise) return;
@@ -1094,7 +1243,7 @@ class ScrcpyDeviceAdapter {
1094
1243
  await this.applyPendingActionBarrier(manager);
1095
1244
  await manager.prepareFreshFrame();
1096
1245
  if (generation !== this.lifecycleGeneration) {
1097
- await manager.disconnect();
1246
+ await manager.dispose();
1098
1247
  if (this.manager === manager) this.manager = null;
1099
1248
  return;
1100
1249
  }
@@ -1103,7 +1252,7 @@ class ScrcpyDeviceAdapter {
1103
1252
  debugAdapter('Scrcpy freshness recovery completed in background');
1104
1253
  } catch (error) {
1105
1254
  if (manager) {
1106
- await manager.disconnect();
1255
+ await manager.dispose();
1107
1256
  if (this.manager === manager) this.manager = null;
1108
1257
  }
1109
1258
  this.freshnessRecoveryPending = false;
@@ -1118,21 +1267,22 @@ class ScrcpyDeviceAdapter {
1118
1267
  });
1119
1268
  }
1120
1269
  async markActionBarrier() {
1270
+ const actionCompletedAtHostUs = this.monotonicTimeUs();
1121
1271
  const manager = this.manager;
1122
- if (!manager?.isConnected()) {
1123
- this.pendingActionBarrier = true;
1124
- return;
1125
- }
1272
+ if (!manager?.isConnected()) return void this.deferActionBarrier(actionCompletedAtHostUs);
1126
1273
  try {
1127
- await manager.setFreshnessBarrier('completed input action');
1128
- this.pendingActionBarrier = false;
1274
+ await manager.setFreshnessBarrier('completed input action', {
1275
+ allowOverAgeForNextCapture: true,
1276
+ hostMonotonicUs: actionCompletedAtHostUs
1277
+ });
1278
+ if (null !== this.pendingActionBarrierAtHostUs && this.pendingActionBarrierAtHostUs <= actionCompletedAtHostUs) this.pendingActionBarrierAtHostUs = null;
1129
1279
  this.clearFailure();
1130
1280
  } catch (error) {
1131
- this.pendingActionBarrier = true;
1281
+ this.deferActionBarrier(actionCompletedAtHostUs);
1132
1282
  this.recordFailure(error);
1133
1283
  debugAdapter(`Unable to mark scrcpy action barrier; disabling this stream: ${error}`);
1134
1284
  try {
1135
- await manager.disconnect();
1285
+ await manager.dispose();
1136
1286
  } catch (disconnectError) {
1137
1287
  debugAdapter(`Error disconnecting scrcpy after barrier failure: ${disconnectError}`);
1138
1288
  } finally{
@@ -1165,14 +1315,16 @@ class ScrcpyDeviceAdapter {
1165
1315
  async disconnect() {
1166
1316
  this.lifecycleGeneration += 1;
1167
1317
  this.freshnessRecoveryPending = false;
1168
- this.pendingActionBarrier = false;
1318
+ this.pendingActionBarrierAtHostUs = null;
1169
1319
  for (const unsubscribe of this.keyframeUnsubscribers.values())unsubscribe();
1170
1320
  this.keyframeUnsubscribers.clear();
1171
1321
  this.keyframeListeners.clear();
1172
1322
  if (this.recoveryPromise) await this.recoveryPromise.catch(()=>{});
1323
+ if (this.freshnessRestartPromise) await this.freshnessRestartPromise.catch(()=>{});
1324
+ if (this.managerPromise) await this.managerPromise.catch(()=>{});
1173
1325
  if (this.manager) {
1174
1326
  try {
1175
- await this.manager.disconnect();
1327
+ await this.manager.dispose();
1176
1328
  } catch (error) {
1177
1329
  debugAdapter(`Error disconnecting scrcpy: ${error}`);
1178
1330
  }
@@ -1186,26 +1338,30 @@ class ScrcpyDeviceAdapter {
1186
1338
  _define_property(this, "scrcpyConfig", void 0);
1187
1339
  _define_property(this, "resolveAdbServerEndpoint", void 0);
1188
1340
  _define_property(this, "manager", void 0);
1341
+ _define_property(this, "managerPromise", void 0);
1189
1342
  _define_property(this, "resolvedConfig", void 0);
1190
1343
  _define_property(this, "lastError", void 0);
1191
1344
  _define_property(this, "retryAfter", void 0);
1192
1345
  _define_property(this, "freshnessRecoveryPending", void 0);
1346
+ _define_property(this, "freshnessRestartPromise", void 0);
1193
1347
  _define_property(this, "recoveryPromise", void 0);
1194
1348
  _define_property(this, "lifecycleGeneration", void 0);
1195
- _define_property(this, "pendingActionBarrier", void 0);
1349
+ _define_property(this, "pendingActionBarrierAtHostUs", void 0);
1196
1350
  _define_property(this, "keyframeListeners", void 0);
1197
1351
  _define_property(this, "keyframeUnsubscribers", void 0);
1198
1352
  this.deviceId = deviceId;
1199
1353
  this.scrcpyConfig = scrcpyConfig;
1200
1354
  this.resolveAdbServerEndpoint = resolveAdbServerEndpoint;
1201
1355
  this.manager = null;
1356
+ this.managerPromise = null;
1202
1357
  this.resolvedConfig = null;
1203
1358
  this.lastError = null;
1204
1359
  this.retryAfter = null;
1205
1360
  this.freshnessRecoveryPending = false;
1361
+ this.freshnessRestartPromise = null;
1206
1362
  this.recoveryPromise = null;
1207
1363
  this.lifecycleGeneration = 0;
1208
- this.pendingActionBarrier = false;
1364
+ this.pendingActionBarrierAtHostUs = null;
1209
1365
  this.keyframeListeners = new Set();
1210
1366
  this.keyframeUnsubscribers = new Map();
1211
1367
  }
@@ -2056,9 +2212,9 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
2056
2212
  }
2057
2213
  async screenshotBase64() {
2058
2214
  debugDevice('screenshotBase64 begin');
2059
- const screenshotStrategy = this.options?.screenshotStrategy || globalConfigManager.getEnvConfigValue(MIDSCENE_ANDROID_SCREENSHOT_STRATEGY) || SCREENSHOT_STRATEGY_AUTO;
2060
- if (screenshotStrategy === SCREENSHOT_STRATEGY_ALWAYS_YADB) return this.screenshotBase64ViaYadb();
2061
2215
  const adapter = this.getScrcpyAdapter();
2216
+ const screenshotStrategy = this.options?.screenshotStrategy || globalConfigManager.getEnvConfigValue(MIDSCENE_ANDROID_SCREENSHOT_STRATEGY) || SCREENSHOT_STRATEGY_AUTO;
2217
+ if (screenshotStrategy === SCREENSHOT_STRATEGY_ALWAYS_YADB) return adapter.prepareFallbackScreenshot(await this.screenshotBase64ViaYadb());
2062
2218
  let scrcpyDeviceInfo = null;
2063
2219
  if (adapter.isEnabled()) try {
2064
2220
  debugDevice('Attempting scrcpy screenshot...');
@@ -2067,7 +2223,7 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
2067
2223
  debugDevice('screenshotBase64 end (scrcpy mode)');
2068
2224
  return result;
2069
2225
  } catch (error) {
2070
- warnDevice(`Scrcpy screenshot failed, falling back to standard ADB method. This may be caused by transport backlog. ${scrcpy_manager.Xr}\nError: ${error}`);
2226
+ warnDevice(`Scrcpy screenshot failed, falling back to standard ADB method.\nError: ${error}`);
2071
2227
  }
2072
2228
  const adb = await this.getAdb();
2073
2229
  let screenshotBuffer;
@@ -2109,14 +2265,14 @@ ${Object.keys(size).filter((key)=>size[key]).map((key)=>` ${key} size: ${size[k
2109
2265
  });
2110
2266
  debugDevice('screenshotBase64 end (fallback)');
2111
2267
  if (scrcpyDeviceInfo) adapter.recoverAfterAdbScreenshot(scrcpyDeviceInfo);
2112
- return result;
2268
+ return adapter.prepareFallbackScreenshot(result);
2113
2269
  }
2114
2270
  if (!screenshotBuffer) throw new Error('Failed to capture screenshot: all methods failed');
2115
2271
  debugDevice('Converting to base64');
2116
2272
  const result = createImgBase64ByFormat('png', screenshotBuffer.toString('base64'));
2117
2273
  debugDevice('screenshotBase64 end');
2118
2274
  if (scrcpyDeviceInfo) adapter.recoverAfterAdbScreenshot(scrcpyDeviceInfo);
2119
- return result;
2275
+ return adapter.prepareFallbackScreenshot(result);
2120
2276
  }
2121
2277
  async captureScreenshotBase64FromDeviceFile(label, capture) {
2122
2278
  const adb = await this.getAdb();
@@ -3134,12 +3290,18 @@ function agent_tools_define_property(obj, key, value) {
3134
3290
  const debug = (0, logger_.getDebug)('agent-tools:android');
3135
3291
  function adaptAndroidInitArgs(extracted) {
3136
3292
  if (!extracted) return;
3293
+ const scrcpyVideoBitRate = 'number' == typeof extracted.scrcpyVideoBitRate ? extracted.scrcpyVideoBitRate : void 0;
3294
+ const requestedUseScrcpy = 'boolean' == typeof extracted.useScrcpy ? extracted.useScrcpy : void 0;
3295
+ const useScrcpy = void 0 !== scrcpyVideoBitRate ? true : requestedUseScrcpy;
3137
3296
  const initArgs = {
3138
3297
  ...'string' == typeof extracted.deviceId ? {
3139
3298
  deviceId: extracted.deviceId
3140
3299
  } : {},
3141
- ...'boolean' == typeof extracted.useScrcpy ? {
3142
- useScrcpy: extracted.useScrcpy
3300
+ ...void 0 !== useScrcpy ? {
3301
+ useScrcpy
3302
+ } : {},
3303
+ ...void 0 !== scrcpyVideoBitRate ? {
3304
+ scrcpyVideoBitRate
3143
3305
  } : {},
3144
3306
  ...extractAgentBehaviorInitArgs(extracted) ?? {}
3145
3307
  };
@@ -3171,7 +3333,10 @@ class AndroidMidsceneTools extends BaseMidsceneTools {
3171
3333
  ...extractAgentBehaviorInitArgs(initArgs) ?? {},
3172
3334
  ...initArgs?.useScrcpy ? {
3173
3335
  scrcpyConfig: {
3174
- enabled: true
3336
+ enabled: true,
3337
+ ...void 0 !== initArgs.scrcpyVideoBitRate ? {
3338
+ videoBitRate: initArgs.scrcpyVideoBitRate
3339
+ } : {}
3175
3340
  }
3176
3341
  } : {},
3177
3342
  ...reportOptions ?? {}
@@ -3229,6 +3394,7 @@ class AndroidMidsceneTools extends BaseMidsceneTools {
3229
3394
  shape: {
3230
3395
  deviceId: z.string().optional().describe('Android device ID (from adb devices)'),
3231
3396
  useScrcpy: z.boolean().optional().describe('Enable scrcpy accelerated screenshots'),
3397
+ scrcpyVideoBitRate: z.number().int().positive().optional().describe('scrcpy H.264 video bitrate in bits per second; providing this option also enables scrcpy. Default with --use-scrcpy: 100000000 (100 Mbps). For constrained remote links, start with 4000000 (4 Mbps).'),
3232
3398
  ...agentBehaviorInitArgShape
3233
3399
  },
3234
3400
  cli: {