@g9tm/app-diagnostic-lynx 0.0.0-stage → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Native-Validation-Pending.md +33 -0
- package/README.md +42 -3
- package/android/build.gradle.kts +45 -0
- package/android/consumer-rules.pro +2 -0
- package/android/src/main/AndroidManifest.xml +5 -0
- package/android/src/main/java/com/g9tm/appdiagnostic/G9AppDiagnosticHost.kt +131 -0
- package/android/src/main/java/com/g9tm/appdiagnostic/G9AppDiagnosticModule.kt +224 -0
- package/android/src/main/java/com/g9tm/appdiagnostic/RemoteTouch.kt +67 -0
- package/android/src/main/java/com/g9tm/appdiagnostic/ScreenRecorder.kt +250 -0
- package/android/src/main/java/com/g9tm/appdiagnostic/TouchWatcher.kt +126 -0
- package/android/src/main/java/com/g9tm/appdiagnostic/VoiceRecorder.kt +82 -0
- package/android/src/main/java/com/g9tm/appdiagnostic/WindowCapture.kt +92 -0
- package/android/src/main/java/com/g9tm/appdiagnostic/core/TouchCore.kt +175 -0
- package/android/src/main/java/com/g9tm/appdiagnostic/generated/G9AppDiagnosticModuleSpec.java +45 -0
- package/generated/G9AppDiagnosticModule.ts +70 -0
- package/lynx.lib.json +12 -0
- package/package.json +39 -6
- package/src/index.ts +157 -0
- package/types/g9-app-diagnostic-module.d.ts +57 -0
|
@@ -0,0 +1,250 @@
|
|
|
1
|
+
package com.g9tm.appdiagnostic
|
|
2
|
+
|
|
3
|
+
import android.app.Activity
|
|
4
|
+
import android.graphics.Color
|
|
5
|
+
import android.media.MediaCodec
|
|
6
|
+
import android.media.MediaCodecInfo
|
|
7
|
+
import android.media.MediaCodecList
|
|
8
|
+
import android.media.MediaFormat
|
|
9
|
+
import android.media.MediaMuxer
|
|
10
|
+
import android.os.SystemClock
|
|
11
|
+
import android.util.Log
|
|
12
|
+
import android.view.Surface
|
|
13
|
+
import java.io.File
|
|
14
|
+
import kotlinx.coroutines.CoroutineScope
|
|
15
|
+
import kotlinx.coroutines.Dispatchers
|
|
16
|
+
import kotlinx.coroutines.Job
|
|
17
|
+
import kotlinx.coroutines.SupervisorJob
|
|
18
|
+
import kotlinx.coroutines.delay
|
|
19
|
+
import kotlinx.coroutines.isActive
|
|
20
|
+
import kotlinx.coroutines.launch
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* The tester's screen recording (G9AppDiagnostic.Maui TesterScreenRecorder + AndroidTesterVideoEncoder): the app's OWN window — the
|
|
24
|
+
* same composited capture as the screenshot (device/WindowCapture, map included) — into an H.264 MP4.
|
|
25
|
+
* No MediaProjection: no system consent dialog, nothing of other apps in the video, no foreground service.
|
|
26
|
+
*
|
|
27
|
+
* - The rate is ADAPTIVE and bounded (MAUI 6.2.12): it starts at the caller's ceiling (the fleet's
|
|
28
|
+
* `tester_capture_max_fps`, default 10, at most 20) and lowers itself — never below 4 — the moment a frame costs
|
|
29
|
+
* most of its slot, climbing back one step after a run of cheap frames: an app slowed by its own recorder records a
|
|
30
|
+
* slowness that is not the defect. Frames carry their own timestamps (the Surface's), so a lower rate is a slower
|
|
31
|
+
* video, never a faster one. Long edge ≤ 960 px, 2 Mbit/s, a key frame every second (a reviewer scrubs).
|
|
32
|
+
* - The loop SKIPS a slot that a slow capture overran instead of queueing (a slow device makes a shorter video, not a
|
|
33
|
+
* backlog and an OOM).
|
|
34
|
+
* - Geometry is locked at the first frame (H.264 cannot change size mid-stream): a rotation letterboxes.
|
|
35
|
+
* - 10 minutes at most: the recording finalises itself and waits for [stop] to hand it over.
|
|
36
|
+
* - Frames reach the encoder through its input Surface with `lockHardwareCanvas` — a bitmap drawn on a canvas, so no
|
|
37
|
+
* YUV conversion and no EGL context (MAUI's ByteBuffer path needed both colour-format negotiation and a converter).
|
|
38
|
+
*
|
|
39
|
+
* TWIN: ios/src/G9AppDiagnosticMedia.m (AVAssetWriter fed from drawViewHierarchyInRect at the same cadence and caps).
|
|
40
|
+
*/
|
|
41
|
+
internal object ScreenRecorder {
|
|
42
|
+
private const val TAG = "G9AppDiagnostic"
|
|
43
|
+
private const val MIN_FPS = 4
|
|
44
|
+
private const val MAX_FPS = 20
|
|
45
|
+
/** The encoder's nominal rate for [supported] (any value works: the probe asks for an encoder, not a rate). */
|
|
46
|
+
private const val PROBE_FPS = 10
|
|
47
|
+
private const val MAX_EDGE = 960
|
|
48
|
+
private const val BIT_RATE = 2_000_000
|
|
49
|
+
private const val MAX_DURATION_MS = 10 * 60_000L
|
|
50
|
+
private const val MIME = MediaFormat.MIMETYPE_VIDEO_AVC
|
|
51
|
+
|
|
52
|
+
class Result(val path: String, val bytes: Long, val durationMs: Long, val width: Int, val height: Int, val capped: Boolean)
|
|
53
|
+
|
|
54
|
+
private class Session(
|
|
55
|
+
val file: File,
|
|
56
|
+
val codec: MediaCodec,
|
|
57
|
+
val surface: Surface,
|
|
58
|
+
val muxer: MediaMuxer,
|
|
59
|
+
val width: Int,
|
|
60
|
+
val height: Int,
|
|
61
|
+
val startedAt: Long,
|
|
62
|
+
) {
|
|
63
|
+
var track = -1
|
|
64
|
+
var muxing = false
|
|
65
|
+
var frames = 0
|
|
66
|
+
var loop: Job? = null
|
|
67
|
+
var ceiling = MIN_FPS
|
|
68
|
+
var fps = MIN_FPS
|
|
69
|
+
@Volatile var capped = false
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Default)
|
|
73
|
+
private val lock = Any()
|
|
74
|
+
private var session: Session? = null
|
|
75
|
+
|
|
76
|
+
/** A recording the 10-minute cap finished, waiting for [stop]. */
|
|
77
|
+
private var finished: Result? = null
|
|
78
|
+
|
|
79
|
+
val isRecording: Boolean
|
|
80
|
+
get() = synchronized(lock) { session != null }
|
|
81
|
+
|
|
82
|
+
/** An AVC encoder that takes Surface input exists (every device in practice; the button hides otherwise). */
|
|
83
|
+
fun supported(): Boolean =
|
|
84
|
+
try {
|
|
85
|
+
val format = MediaFormat.createVideoFormat(MIME, 960, 544).apply {
|
|
86
|
+
setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface)
|
|
87
|
+
setInteger(MediaFormat.KEY_BIT_RATE, BIT_RATE)
|
|
88
|
+
setInteger(MediaFormat.KEY_FRAME_RATE, PROBE_FPS)
|
|
89
|
+
setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1)
|
|
90
|
+
}
|
|
91
|
+
MediaCodecList(MediaCodecList.REGULAR_CODECS).findEncoderForFormat(format) != null
|
|
92
|
+
} catch (_: Throwable) {
|
|
93
|
+
false
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** Output size: the window scaled to a ≤ 960 px long edge, both sides even (H.264). */
|
|
97
|
+
internal fun outputSize(width: Int, height: Int): Pair<Int, Int> {
|
|
98
|
+
val scale = minOf(1.0, MAX_EDGE.toDouble() / maxOf(width, height))
|
|
99
|
+
val w = ((width * scale).toInt() / 2 * 2).coerceAtLeast(2)
|
|
100
|
+
val h = ((height * scale).toInt() / 2 * 2).coerceAtLeast(2)
|
|
101
|
+
return w to h
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
suspend fun start(dir: File, maxFps: Int) {
|
|
105
|
+
val ceiling = maxFps.coerceIn(MIN_FPS, MAX_FPS)
|
|
106
|
+
synchronized(lock) {
|
|
107
|
+
if (session != null) throw G9DiagnosticException("busy", "A screen recording is already running")
|
|
108
|
+
finished = null
|
|
109
|
+
}
|
|
110
|
+
val activity = G9AppDiagnosticHost.awaitActivity()
|
|
111
|
+
val (ww, wh) = WindowCapture.windowSize(activity)
|
|
112
|
+
if (ww <= 0 || wh <= 0) throw G9DiagnosticException("unavailable", "The window is not laid out")
|
|
113
|
+
val (width, height) = outputSize(ww, wh)
|
|
114
|
+
dir.mkdirs()
|
|
115
|
+
val file = File(dir, "screen-${System.currentTimeMillis()}.mp4")
|
|
116
|
+
val format = MediaFormat.createVideoFormat(MIME, width, height).apply {
|
|
117
|
+
setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface)
|
|
118
|
+
setInteger(MediaFormat.KEY_BIT_RATE, BIT_RATE)
|
|
119
|
+
// The encoder is told the CEILING (its bitrate / key-frame budget); the actual rate may be lower.
|
|
120
|
+
setInteger(MediaFormat.KEY_FRAME_RATE, ceiling)
|
|
121
|
+
setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1)
|
|
122
|
+
}
|
|
123
|
+
val codec = MediaCodec.createEncoderByType(MIME)
|
|
124
|
+
try {
|
|
125
|
+
codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE)
|
|
126
|
+
} catch (e: Exception) {
|
|
127
|
+
codec.release()
|
|
128
|
+
throw G9DiagnosticException("unavailable", "No encoder for ${width}x$height: ${e.message}", e)
|
|
129
|
+
}
|
|
130
|
+
val surface = codec.createInputSurface()
|
|
131
|
+
codec.start()
|
|
132
|
+
val s = Session(file, codec, surface, MediaMuxer(file.path, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4), width, height, SystemClock.elapsedRealtime())
|
|
133
|
+
s.ceiling = ceiling
|
|
134
|
+
s.fps = ceiling
|
|
135
|
+
synchronized(lock) { session = s }
|
|
136
|
+
s.loop = scope.launch { captureLoop(activity, s) }
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
private suspend fun captureLoop(activity: Activity, s: Session) {
|
|
140
|
+
var slotMs = 1000L / s.fps
|
|
141
|
+
var next = SystemClock.elapsedRealtime()
|
|
142
|
+
var costMs = 0.0 // exponential moving average of one frame's cost
|
|
143
|
+
var cheapStreak = 0
|
|
144
|
+
while (scope.isActive && synchronized(lock) { session === s }) {
|
|
145
|
+
if (SystemClock.elapsedRealtime() - s.startedAt >= MAX_DURATION_MS) {
|
|
146
|
+
s.capped = true
|
|
147
|
+
synchronized(lock) { if (session === s) session = null }
|
|
148
|
+
finished = finish(s)
|
|
149
|
+
return
|
|
150
|
+
}
|
|
151
|
+
val frameStarted = SystemClock.elapsedRealtime()
|
|
152
|
+
try {
|
|
153
|
+
val frame = WindowCapture.capture(activity, s.width, s.height)
|
|
154
|
+
val canvas = s.surface.lockHardwareCanvas()
|
|
155
|
+
try {
|
|
156
|
+
canvas.drawColor(Color.BLACK)
|
|
157
|
+
canvas.drawBitmap(frame, 0f, 0f, null)
|
|
158
|
+
} finally {
|
|
159
|
+
s.surface.unlockCanvasAndPost(canvas)
|
|
160
|
+
}
|
|
161
|
+
frame.recycle()
|
|
162
|
+
s.frames++
|
|
163
|
+
drain(s, endOfStream = false)
|
|
164
|
+
} catch (e: Exception) {
|
|
165
|
+
Log.w(TAG, "screen frame skipped: ${e.message}")
|
|
166
|
+
}
|
|
167
|
+
// Adapt: down at once when a frame eats most of its slot, up one step after a run of cheap ones.
|
|
168
|
+
val elapsed = (SystemClock.elapsedRealtime() - frameStarted).toDouble()
|
|
169
|
+
costMs = if (costMs <= 0.0) elapsed else costMs * 0.7 + elapsed * 0.3
|
|
170
|
+
val slot = 1000.0 / s.fps
|
|
171
|
+
if (costMs > slot * 0.85 && s.fps > MIN_FPS) {
|
|
172
|
+
s.fps = maxOf(MIN_FPS, Math.floor(1000.0 / (costMs * 1.25)).toInt())
|
|
173
|
+
slotMs = 1000L / s.fps
|
|
174
|
+
cheapStreak = 0
|
|
175
|
+
} else if (costMs < slot * 0.5 && s.fps < s.ceiling && ++cheapStreak >= s.fps * 3) {
|
|
176
|
+
s.fps++
|
|
177
|
+
slotMs = 1000L / s.fps
|
|
178
|
+
cheapStreak = 0
|
|
179
|
+
}
|
|
180
|
+
next += slotMs
|
|
181
|
+
val now = SystemClock.elapsedRealtime()
|
|
182
|
+
if (next < now) next = now + slotMs - ((now - next) % slotMs) // overran: skip the missed slots
|
|
183
|
+
delay(next - now)
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/** Writes whatever the encoder produced; with [endOfStream], until the end-of-stream buffer. */
|
|
188
|
+
private fun drain(s: Session, endOfStream: Boolean) {
|
|
189
|
+
val info = MediaCodec.BufferInfo()
|
|
190
|
+
while (true) {
|
|
191
|
+
val index = s.codec.dequeueOutputBuffer(info, if (endOfStream) 10_000 else 0)
|
|
192
|
+
when {
|
|
193
|
+
index == MediaCodec.INFO_TRY_AGAIN_LATER -> if (!endOfStream) return
|
|
194
|
+
index == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED -> {
|
|
195
|
+
if (!s.muxing) {
|
|
196
|
+
s.track = s.muxer.addTrack(s.codec.outputFormat)
|
|
197
|
+
s.muxer.start()
|
|
198
|
+
s.muxing = true
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
index >= 0 -> {
|
|
202
|
+
val buffer = s.codec.getOutputBuffer(index)
|
|
203
|
+
if ((info.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) info.size = 0
|
|
204
|
+
if (buffer != null && info.size > 0 && s.muxing) {
|
|
205
|
+
buffer.position(info.offset)
|
|
206
|
+
buffer.limit(info.offset + info.size)
|
|
207
|
+
s.muxer.writeSampleData(s.track, buffer, info)
|
|
208
|
+
}
|
|
209
|
+
s.codec.releaseOutputBuffer(index, false)
|
|
210
|
+
if ((info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) return
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
private fun finish(s: Session): Result? {
|
|
217
|
+
val durationMs = SystemClock.elapsedRealtime() - s.startedAt
|
|
218
|
+
// What the adaptive rate settled on — the way to judge `tester_capture_max_fps` on a real device.
|
|
219
|
+
Log.i(TAG, "screen recording: ${s.frames} frames in ${durationMs / 1000.0} s, last rate ${s.fps} fps, ceiling ${s.ceiling}")
|
|
220
|
+
var ok = s.frames > 0
|
|
221
|
+
try {
|
|
222
|
+
s.codec.signalEndOfInputStream()
|
|
223
|
+
drain(s, endOfStream = true)
|
|
224
|
+
} catch (e: Exception) {
|
|
225
|
+
Log.w(TAG, "screen recording end: ${e.message}")
|
|
226
|
+
ok = false
|
|
227
|
+
}
|
|
228
|
+
try {
|
|
229
|
+
if (s.muxing) s.muxer.stop() else ok = false
|
|
230
|
+
} catch (e: Exception) {
|
|
231
|
+
ok = false
|
|
232
|
+
}
|
|
233
|
+
runCatching { s.muxer.release() }
|
|
234
|
+
runCatching { s.codec.stop() }
|
|
235
|
+
runCatching { s.codec.release() }
|
|
236
|
+
runCatching { s.surface.release() }
|
|
237
|
+
if (!ok || !s.file.isFile || s.file.length() == 0L) {
|
|
238
|
+
s.file.delete()
|
|
239
|
+
return null
|
|
240
|
+
}
|
|
241
|
+
return Result(s.file.path, s.file.length(), durationMs, s.width, s.height, s.capped)
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
/** Stops and hands over the MP4 (null when nothing usable was recorded). */
|
|
245
|
+
suspend fun stop(): Result? {
|
|
246
|
+
val s = synchronized(lock) { session.also { session = null } } ?: return finished.also { finished = null }
|
|
247
|
+
s.loop?.join() // the loop sees `session !== s` after its current frame
|
|
248
|
+
return finish(s)
|
|
249
|
+
}
|
|
250
|
+
}
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
package com.g9tm.appdiagnostic
|
|
2
|
+
|
|
3
|
+
import android.app.Activity
|
|
4
|
+
import android.view.MotionEvent
|
|
5
|
+
import android.view.View
|
|
6
|
+
import android.view.ViewConfiguration
|
|
7
|
+
import android.view.ViewGroup
|
|
8
|
+
import android.view.Window
|
|
9
|
+
import com.g9tm.appdiagnostic.core.TouchCore
|
|
10
|
+
import org.json.JSONObject
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* The tester gesture on Android: every touch of the activity's window, seen BEFORE any view through a wrapper around the
|
|
14
|
+
* window's callback — so a host needs no edit in its activity (MAUI needed `MainActivity.DispatchTouchEvent` to forward).
|
|
15
|
+
* The decisions are [TouchCore]'s; this class only converts events and carries the verdicts out: a CANCEL to the views
|
|
16
|
+
* when a hold begins, a replayed DOWN + POINTER_DOWNs when a held touch turns out to be a pinch or a pan.
|
|
17
|
+
*
|
|
18
|
+
* Off (nothing watched) the wrapper returns at its first line: Tester Mode off costs one field read per touch.
|
|
19
|
+
*
|
|
20
|
+
* TWIN: ios/src/G9AppDiagnosticTouches.m (a recogniser on the key window that cancels the views' touches).
|
|
21
|
+
*/
|
|
22
|
+
internal object TouchWatcher {
|
|
23
|
+
const val EVENT = "g9:diagnostic-touch"
|
|
24
|
+
|
|
25
|
+
@Volatile private var core: TouchCore? = null
|
|
26
|
+
|
|
27
|
+
private fun coreFor(activity: Activity): TouchCore =
|
|
28
|
+
core ?: synchronized(this) {
|
|
29
|
+
core ?: TouchCore(maxOf(16, ViewConfiguration.get(activity).scaledTouchSlop * 2).toFloat()) { android.os.SystemClock.uptimeMillis() }.also { core = it }
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** Sets what is watched; the window of the activity on screen is wrapped (once per window). */
|
|
33
|
+
fun configure(activity: Activity?, taps: Boolean, touches: Boolean, consume: Boolean, excludeSurfaces: Boolean) {
|
|
34
|
+
val c = core ?: activity?.let { coreFor(it) } ?: return
|
|
35
|
+
c.taps = taps
|
|
36
|
+
c.touches = touches
|
|
37
|
+
c.consume = consume
|
|
38
|
+
c.excludeSurfaces = excludeSurfaces
|
|
39
|
+
activity?.let { attach(it) }
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** Wraps [activity]'s window callback (an activity recreated by a rotation gets wrapped again). Main thread. */
|
|
43
|
+
fun attach(activity: Activity) {
|
|
44
|
+
val window = activity.window ?: return
|
|
45
|
+
val callback = window.callback ?: return
|
|
46
|
+
if (callback is Watching) return
|
|
47
|
+
val c = coreFor(activity)
|
|
48
|
+
c.isExcluded = { x, y -> isOverExcluded(activity, x, y) }
|
|
49
|
+
window.callback = Watching(callback, activity)
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
private class Watching(private val inner: Window.Callback, private val activity: Activity) : Window.Callback by inner {
|
|
53
|
+
override fun dispatchTouchEvent(event: MotionEvent): Boolean {
|
|
54
|
+
val c = core
|
|
55
|
+
if (c == null || !c.watching) return inner.dispatchTouchEvent(event)
|
|
56
|
+
val result =
|
|
57
|
+
try {
|
|
58
|
+
c.onEvent(sample(event))
|
|
59
|
+
} catch (_: Throwable) {
|
|
60
|
+
return inner.dispatchTouchEvent(event) // a watcher never breaks a touch
|
|
61
|
+
}
|
|
62
|
+
for (e in result.emits) emit(e)
|
|
63
|
+
return when (result.verdict) {
|
|
64
|
+
TouchCore.Verdict.PASS -> inner.dispatchTouchEvent(event)
|
|
65
|
+
TouchCore.Verdict.CONSUME -> true
|
|
66
|
+
TouchCore.Verdict.CANCEL_THEN_CONSUME -> {
|
|
67
|
+
val cancel = MotionEvent.obtain(event)
|
|
68
|
+
cancel.action = MotionEvent.ACTION_CANCEL
|
|
69
|
+
inner.dispatchTouchEvent(cancel)
|
|
70
|
+
cancel.recycle()
|
|
71
|
+
true
|
|
72
|
+
}
|
|
73
|
+
TouchCore.Verdict.REPLAY -> {
|
|
74
|
+
replay(event)
|
|
75
|
+
true
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** A held gesture back to the views: a DOWN for the first finger, a POINTER_DOWN per further one, where they are NOW. */
|
|
81
|
+
private fun replay(event: MotionEvent) {
|
|
82
|
+
val count = event.pointerCount
|
|
83
|
+
val properties = Array(count) { MotionEvent.PointerProperties().also { p -> event.getPointerProperties(it, p) } }
|
|
84
|
+
val coords = Array(count) { MotionEvent.PointerCoords().also { p -> event.getPointerCoords(it, p) } }
|
|
85
|
+
for (fingers in 1..count) {
|
|
86
|
+
val action = if (fingers == 1) MotionEvent.ACTION_DOWN else MotionEvent.ACTION_POINTER_DOWN or ((fingers - 1) shl MotionEvent.ACTION_POINTER_INDEX_SHIFT)
|
|
87
|
+
val synthetic =
|
|
88
|
+
MotionEvent.obtain(
|
|
89
|
+
event.downTime, event.eventTime, action, fingers, properties.copyOf(fingers).requireNoNulls(), coords.copyOf(fingers).requireNoNulls(),
|
|
90
|
+
event.metaState, event.buttonState, event.xPrecision, event.yPrecision, event.deviceId, event.edgeFlags, event.source, event.flags,
|
|
91
|
+
)
|
|
92
|
+
inner.dispatchTouchEvent(synthetic)
|
|
93
|
+
synthetic.recycle()
|
|
94
|
+
}
|
|
95
|
+
inner.dispatchTouchEvent(event)
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
private fun emit(e: TouchCore.Emit) {
|
|
99
|
+
val root = activity.window?.decorView ?: return
|
|
100
|
+
val w = root.width.toFloat()
|
|
101
|
+
val h = root.height.toFloat()
|
|
102
|
+
if (w <= 0f || h <= 0f) return
|
|
103
|
+
val json = JSONObject().put("type", e.type).put("fingers", e.fingers).put("x", (e.x / w).toDouble()).put("y", (e.y / h).toDouble()).put("excluded", e.excluded)
|
|
104
|
+
G9AppDiagnosticHost.emit(EVENT, json.toString())
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
private fun sample(event: MotionEvent): TouchCore.Sample =
|
|
109
|
+
TouchCore.Sample(event.actionMasked, List(event.pointerCount) { TouchCore.Pointer(event.getPointerId(it), event.getX(it), event.getY(it)) })
|
|
110
|
+
|
|
111
|
+
/** A visible excluded view under the window point. A first finger is rare enough to walk the tree. */
|
|
112
|
+
private fun isOverExcluded(activity: Activity, x: Float, y: Float): Boolean {
|
|
113
|
+
val root = activity.window?.decorView ?: return false
|
|
114
|
+
fun walk(view: View): Boolean {
|
|
115
|
+
if (!view.isShown) return false
|
|
116
|
+
if (G9AppDiagnosticHost.isExcludedView(view)) {
|
|
117
|
+
val at = IntArray(2)
|
|
118
|
+
view.getLocationInWindow(at)
|
|
119
|
+
return x >= at[0] && x <= at[0] + view.width && y >= at[1] && y <= at[1] + view.height
|
|
120
|
+
}
|
|
121
|
+
if (view is ViewGroup) for (i in 0 until view.childCount) if (walk(view.getChildAt(i))) return true
|
|
122
|
+
return false
|
|
123
|
+
}
|
|
124
|
+
return walk(root)
|
|
125
|
+
}
|
|
126
|
+
}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
package com.g9tm.appdiagnostic
|
|
2
|
+
|
|
3
|
+
import android.content.pm.PackageManager
|
|
4
|
+
import android.media.MediaRecorder
|
|
5
|
+
import android.os.Build
|
|
6
|
+
import android.os.SystemClock
|
|
7
|
+
import java.io.File
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* The tester's spoken note (G9AppDiagnostic.Maui AndroidVoiceRecorder): the microphone into AAC in an MP4 container (`.m4a`), mono,
|
|
11
|
+
* 44.1 kHz, 96 kbit/s — every player and the Diagnostic server's preview open it. 10 minutes at most (MediaRecorder's
|
|
12
|
+
* own max duration: the file is finalised and kept). The RECORD_AUDIO grant is asked by the module before [start] (the package's manifest declares it).
|
|
13
|
+
*
|
|
14
|
+
* TWIN: ios/src/G9AppDiagnosticMedia.m (AVAudioRecorder, the same format and cap).
|
|
15
|
+
*/
|
|
16
|
+
internal object VoiceRecorder {
|
|
17
|
+
private const val MAX_DURATION_MS = 10 * 60_000
|
|
18
|
+
|
|
19
|
+
class Result(val path: String, val bytes: Long, val durationMs: Long)
|
|
20
|
+
|
|
21
|
+
private val lock = Any()
|
|
22
|
+
private var recorder: MediaRecorder? = null
|
|
23
|
+
private var file: File? = null
|
|
24
|
+
private var startedAt = 0L
|
|
25
|
+
|
|
26
|
+
val isRecording: Boolean
|
|
27
|
+
get() = synchronized(lock) { recorder != null }
|
|
28
|
+
|
|
29
|
+
fun supported(): Boolean = G9AppDiagnosticHost.app.packageManager.hasSystemFeature(PackageManager.FEATURE_MICROPHONE)
|
|
30
|
+
|
|
31
|
+
fun start(dir: File) =
|
|
32
|
+
synchronized(lock) {
|
|
33
|
+
if (recorder != null) throw G9DiagnosticException("busy", "A voice note is already being recorded")
|
|
34
|
+
dir.mkdirs()
|
|
35
|
+
val target = File(dir, "voice-${System.currentTimeMillis()}.m4a")
|
|
36
|
+
@Suppress("DEPRECATION")
|
|
37
|
+
val r = if (Build.VERSION.SDK_INT >= 31) MediaRecorder(G9AppDiagnosticHost.app) else MediaRecorder()
|
|
38
|
+
try {
|
|
39
|
+
r.setAudioSource(MediaRecorder.AudioSource.MIC)
|
|
40
|
+
r.setOutputFormat(MediaRecorder.OutputFormat.MPEG_4)
|
|
41
|
+
r.setAudioEncoder(MediaRecorder.AudioEncoder.AAC)
|
|
42
|
+
r.setAudioChannels(1)
|
|
43
|
+
r.setAudioSamplingRate(44_100)
|
|
44
|
+
r.setAudioEncodingBitRate(96_000)
|
|
45
|
+
r.setMaxDuration(MAX_DURATION_MS)
|
|
46
|
+
r.setOutputFile(target.path)
|
|
47
|
+
r.prepare()
|
|
48
|
+
r.start()
|
|
49
|
+
} catch (e: Exception) {
|
|
50
|
+
r.release()
|
|
51
|
+
target.delete()
|
|
52
|
+
throw G9DiagnosticException("unavailable", "The microphone could not start: ${e.message}", e)
|
|
53
|
+
}
|
|
54
|
+
recorder = r
|
|
55
|
+
file = target
|
|
56
|
+
startedAt = SystemClock.elapsedRealtime()
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** Stops and hands over the note (null when nothing was recorded — a stop right after start). */
|
|
60
|
+
fun stop(): Result? =
|
|
61
|
+
synchronized(lock) {
|
|
62
|
+
val r = recorder ?: return null
|
|
63
|
+
val f = file
|
|
64
|
+
val duration = SystemClock.elapsedRealtime() - startedAt
|
|
65
|
+
recorder = null
|
|
66
|
+
file = null
|
|
67
|
+
val ok =
|
|
68
|
+
try {
|
|
69
|
+
r.stop()
|
|
70
|
+
true
|
|
71
|
+
} catch (_: RuntimeException) {
|
|
72
|
+
false // stop() throws when no audio was captured yet: the file is unusable
|
|
73
|
+
} finally {
|
|
74
|
+
r.release()
|
|
75
|
+
}
|
|
76
|
+
if (!ok || f == null || !f.isFile || f.length() == 0L) {
|
|
77
|
+
f?.delete()
|
|
78
|
+
return null
|
|
79
|
+
}
|
|
80
|
+
Result(f.path, f.length(), duration)
|
|
81
|
+
}
|
|
82
|
+
}
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
package com.g9tm.appdiagnostic
|
|
2
|
+
|
|
3
|
+
import android.app.Activity
|
|
4
|
+
import android.graphics.Bitmap
|
|
5
|
+
import android.graphics.Canvas
|
|
6
|
+
import android.graphics.Color
|
|
7
|
+
import android.graphics.Rect
|
|
8
|
+
import android.os.Handler
|
|
9
|
+
import android.os.Looper
|
|
10
|
+
import android.view.PixelCopy
|
|
11
|
+
import android.view.SurfaceView
|
|
12
|
+
import android.view.View
|
|
13
|
+
import android.view.ViewGroup
|
|
14
|
+
import kotlin.coroutines.resume
|
|
15
|
+
import kotlinx.coroutines.Dispatchers
|
|
16
|
+
import kotlinx.coroutines.suspendCancellableCoroutine
|
|
17
|
+
import kotlinx.coroutines.withContext
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* The app window as the user sees it, at any output size: the screenshot (G9AppDiagnosticModule.screenshot) and the tester screen
|
|
21
|
+
* recording (media/ScreenRecorder, 4 times a second) both read it here.
|
|
22
|
+
*
|
|
23
|
+
* PixelCopy(Window) reads the window's own surface, where a SurfaceView (a map, a video) is only a transparent hole:
|
|
24
|
+
* each visible SurfaceView is copied too and drawn UNDER the window, so the map shows through its hole and the chrome
|
|
25
|
+
* above it stays on top. PixelCopy scales into the destination bitmap, so a small output never allocates a full-size
|
|
26
|
+
* frame first (a recording at 960 px reads 960 px, not 2400).
|
|
27
|
+
*
|
|
28
|
+
* TWIN: ios/src/G9AppDiagnosticCapture.m (drawViewHierarchyInRect, which includes the Metal map layer).
|
|
29
|
+
*/
|
|
30
|
+
internal object WindowCapture {
|
|
31
|
+
/** The decor view's size in pixels, read on the main thread (0 × 0 before layout). */
|
|
32
|
+
suspend fun windowSize(activity: Activity): Pair<Int, Int> =
|
|
33
|
+
withContext(Dispatchers.Main) { activity.window.decorView.let { it.width to it.height } }
|
|
34
|
+
|
|
35
|
+
/** The window composited at [outWidth] × [outHeight] (the caller keeps the aspect ratio). */
|
|
36
|
+
suspend fun capture(activity: Activity, outWidth: Int, outHeight: Int): Bitmap {
|
|
37
|
+
val (width, height) = windowSize(activity)
|
|
38
|
+
if (width <= 0 || height <= 0) throw G9DiagnosticException("unavailable", "The window is not laid out")
|
|
39
|
+
val sx = outWidth.toFloat() / width
|
|
40
|
+
val sy = outHeight.toFloat() / height
|
|
41
|
+
val window = copyWindow(activity, outWidth, outHeight)
|
|
42
|
+
val surfaces = withContext(Dispatchers.Main) { surfaceViews(activity.window.decorView) }
|
|
43
|
+
if (surfaces.isEmpty()) return window
|
|
44
|
+
val frame = Bitmap.createBitmap(outWidth, outHeight, Bitmap.Config.ARGB_8888)
|
|
45
|
+
val canvas = Canvas(frame)
|
|
46
|
+
canvas.drawColor(Color.BLACK)
|
|
47
|
+
for ((view, x, y) in surfaces) {
|
|
48
|
+
val w = (view.width * sx).toInt().coerceAtLeast(1)
|
|
49
|
+
val h = (view.height * sy).toInt().coerceAtLeast(1)
|
|
50
|
+
val copy = copySurface(view, w, h) ?: continue
|
|
51
|
+
canvas.drawBitmap(copy, null, Rect((x * sx).toInt(), (y * sy).toInt(), (x * sx).toInt() + w, (y * sy).toInt() + h), null)
|
|
52
|
+
copy.recycle()
|
|
53
|
+
}
|
|
54
|
+
canvas.drawBitmap(window, 0f, 0f, null)
|
|
55
|
+
window.recycle()
|
|
56
|
+
return frame
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
private suspend fun copyWindow(activity: Activity, width: Int, height: Int): Bitmap =
|
|
60
|
+
suspendCancellableCoroutine { cont ->
|
|
61
|
+
val bitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888)
|
|
62
|
+
PixelCopy.request(activity.window, bitmap, { rc ->
|
|
63
|
+
if (rc == PixelCopy.SUCCESS) cont.resume(bitmap) else cont.cancel(G9DiagnosticException("io", "PixelCopy failed ($rc)"))
|
|
64
|
+
}, Handler(Looper.getMainLooper()))
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
private suspend fun copySurface(view: SurfaceView, width: Int, height: Int): Bitmap? =
|
|
68
|
+
suspendCancellableCoroutine { cont ->
|
|
69
|
+
val surface = view.holder.surface
|
|
70
|
+
if (!surface.isValid || view.width <= 0 || view.height <= 0) {
|
|
71
|
+
cont.resume(null)
|
|
72
|
+
return@suspendCancellableCoroutine
|
|
73
|
+
}
|
|
74
|
+
val bitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888)
|
|
75
|
+
PixelCopy.request(view, bitmap, { rc -> cont.resume(if (rc == PixelCopy.SUCCESS) bitmap else null) }, Handler(Looper.getMainLooper()))
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
private fun surfaceViews(root: View): List<Triple<SurfaceView, Int, Int>> {
|
|
79
|
+
val found = ArrayList<Triple<SurfaceView, Int, Int>>()
|
|
80
|
+
fun walk(v: View) {
|
|
81
|
+
if (v.visibility != View.VISIBLE) return
|
|
82
|
+
if (v is SurfaceView) {
|
|
83
|
+
val at = IntArray(2)
|
|
84
|
+
v.getLocationInWindow(at)
|
|
85
|
+
found += Triple(v, at[0], at[1])
|
|
86
|
+
}
|
|
87
|
+
if (v is ViewGroup) for (i in 0 until v.childCount) walk(v.getChildAt(i))
|
|
88
|
+
}
|
|
89
|
+
walk(root)
|
|
90
|
+
return found
|
|
91
|
+
}
|
|
92
|
+
}
|