agent-device 0.10.0 → 0.10.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -607
- package/dist/src/331.js +3 -3
- package/dist/src/425.js +1 -0
- package/dist/src/bin.js +28 -28
- package/dist/src/core/dispatch.d.ts +2 -0
- package/dist/src/core/session-surface.d.ts +3 -0
- package/dist/src/core/settings-contract.d.ts +2 -1
- package/dist/src/daemon/android-system-dialog.d.ts +11 -0
- package/dist/src/daemon/app-log-ios.d.ts +2 -1
- package/dist/src/daemon/app-log-process.d.ts +1 -1
- package/dist/src/daemon/app-log.d.ts +1 -1
- package/dist/src/daemon/context.d.ts +2 -0
- package/dist/src/daemon/handlers/interaction-common.d.ts +30 -1
- package/dist/src/daemon/handlers/interaction-read.d.ts +14 -0
- package/dist/src/daemon/handlers/interaction-touch.d.ts +45 -0
- package/dist/src/daemon/handlers/interaction.d.ts +2 -0
- package/dist/src/daemon/handlers/record-trace-android.d.ts +18 -0
- package/dist/src/daemon/handlers/record-trace-ios.d.ts +52 -0
- package/dist/src/daemon/handlers/record-trace-recording.d.ts +32 -0
- package/dist/src/daemon/handlers/record-trace.d.ts +2 -7
- package/dist/src/daemon/handlers/snapshot-capture.d.ts +11 -4
- package/dist/src/daemon/record-trace-errors.d.ts +6 -0
- package/dist/src/daemon/recording-gestures.d.ts +3 -0
- package/dist/src/daemon/recording-telemetry.d.ts +20 -0
- package/dist/src/daemon/recording-timing.d.ts +24 -0
- package/dist/src/daemon/request-router.d.ts +6 -0
- package/dist/src/daemon/script-utils.d.ts +1 -0
- package/dist/src/daemon/snapshot-processing.d.ts +1 -0
- package/dist/src/daemon/touch-reference-frame.d.ts +7 -0
- package/dist/src/daemon/types.d.ts +65 -11
- package/dist/src/daemon.js +62 -36
- package/dist/src/platforms/android/index.d.ts +1 -1
- package/dist/src/platforms/android/input-actions.d.ts +5 -0
- package/dist/src/platforms/android/settings.d.ts +1 -1
- package/dist/src/platforms/ios/apps.d.ts +1 -1
- package/dist/src/platforms/ios/macos-helper.d.ts +69 -0
- package/dist/src/platforms/ios/runner-client.d.ts +2 -2
- package/dist/src/platforms/ios/runner-session.d.ts +5 -0
- package/dist/src/platforms/ios/runner-xctestrun.d.ts +3 -1
- package/dist/src/recording/overlay.d.ts +10 -0
- package/dist/src/utils/command-schema.d.ts +2 -0
- package/dist/src/utils/interactors.d.ts +8 -8
- package/dist/src/utils/snapshot-lines.d.ts +5 -2
- package/dist/src/utils/snapshot.d.ts +8 -1
- package/dist/src/utils/text-surface.d.ts +19 -0
- package/dist/src/utils/video.d.ts +9 -0
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +196 -51
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +133 -0
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Lifecycle.swift +1 -1
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +33 -1
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+ScreenRecorder.swift +4 -6
- package/ios-runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests.swift +1 -0
- package/ios-runner/AgentDeviceRunner/RecordingScripts/recording-overlay.swift +571 -0
- package/ios-runner/AgentDeviceRunner/RecordingScripts/recording-trim.swift +140 -0
- package/macos-helper/Package.swift +18 -0
- package/macos-helper/Sources/AgentDeviceMacOSHelper/SnapshotTraversal.swift +543 -0
- package/macos-helper/Sources/AgentDeviceMacOSHelper/main.swift +545 -0
- package/package.json +4 -1
- package/skills/agent-device/SKILL.md +25 -334
- package/skills/agent-device/references/bootstrap-install.md +167 -0
- package/skills/agent-device/references/coordinate-system.md +24 -4
- package/skills/agent-device/references/debugging.md +115 -0
- package/skills/agent-device/references/exploration.md +193 -0
- package/skills/agent-device/references/macos-desktop.md +55 -57
- package/skills/agent-device/references/remote-tenancy.md +56 -47
- package/skills/agent-device/references/verification.md +103 -0
- package/dist/src/274.js +0 -1
- package/dist/src/daemon/handlers/interaction-fill.d.ts +0 -3
- package/dist/src/daemon/handlers/interaction-press.d.ts +0 -3
- package/skills/agent-device/references/batching.md +0 -79
- package/skills/agent-device/references/logs-and-debug.md +0 -113
- package/skills/agent-device/references/perf-metrics.md +0 -53
- package/skills/agent-device/references/permissions.md +0 -70
- package/skills/agent-device/references/session-management.md +0 -101
- package/skills/agent-device/references/snapshot-refs.md +0 -102
- package/skills/agent-device/references/video-recording.md +0 -41
|
@@ -0,0 +1,571 @@
|
|
|
1
|
+
import AppKit
|
|
2
|
+
import AVFoundation
|
|
3
|
+
import Foundation
|
|
4
|
+
import QuartzCore
|
|
5
|
+
|
|
6
|
+
let touchDotColor = NSColor(calibratedRed: 0.20, green: 0.63, blue: 0.98, alpha: 0.48).cgColor
|
|
7
|
+
let touchDotBorderColor = NSColor(calibratedRed: 0.94, green: 0.98, blue: 1.0, alpha: 0.68).cgColor
|
|
8
|
+
let minimumTapVisibility: CFTimeInterval = 0.45
|
|
9
|
+
let minimumSwipeVisibility: CFTimeInterval = 0.5
|
|
10
|
+
let minimumPinchVisibility: CFTimeInterval = 0.5
|
|
11
|
+
let swipeVisibilityTail: CFTimeInterval = 0.16
|
|
12
|
+
let trailOpacityKeyTimes: [NSNumber] = [0.0, 0.08, 0.62, 1.0]
|
|
13
|
+
|
|
14
|
+
struct GestureEnvelope: Decodable {
|
|
15
|
+
let events: [GestureEvent]
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
struct GestureEvent: Decodable {
|
|
19
|
+
let kind: String
|
|
20
|
+
let tMs: Double
|
|
21
|
+
let x: Double
|
|
22
|
+
let y: Double
|
|
23
|
+
let x2: Double?
|
|
24
|
+
let y2: Double?
|
|
25
|
+
let referenceWidth: Double?
|
|
26
|
+
let referenceHeight: Double?
|
|
27
|
+
let durationMs: Double?
|
|
28
|
+
let scale: Double?
|
|
29
|
+
let contentDirection: String?
|
|
30
|
+
let edge: String?
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
enum OverlayError: Error, CustomStringConvertible {
|
|
34
|
+
case invalidArgs(String)
|
|
35
|
+
case missingVideoTrack
|
|
36
|
+
case exportFailed(String)
|
|
37
|
+
|
|
38
|
+
var description: String {
|
|
39
|
+
switch self {
|
|
40
|
+
case .invalidArgs(let message):
|
|
41
|
+
return message
|
|
42
|
+
case .missingVideoTrack:
|
|
43
|
+
return "Input video does not contain a video track."
|
|
44
|
+
case .exportFailed(let message):
|
|
45
|
+
return message
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
do {
|
|
51
|
+
try run()
|
|
52
|
+
} catch {
|
|
53
|
+
fputs("recording-overlay: \(error)\n", stderr)
|
|
54
|
+
exit(1)
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
func run() throws {
|
|
58
|
+
let arguments = Array(CommandLine.arguments.dropFirst())
|
|
59
|
+
let parsedArgs = try parseArguments(arguments)
|
|
60
|
+
let inputURL = URL(fileURLWithPath: parsedArgs.inputPath)
|
|
61
|
+
let outputURL = URL(fileURLWithPath: parsedArgs.outputPath)
|
|
62
|
+
let eventsURL = URL(fileURLWithPath: parsedArgs.eventsPath)
|
|
63
|
+
|
|
64
|
+
if FileManager.default.fileExists(atPath: outputURL.path) {
|
|
65
|
+
try FileManager.default.removeItem(at: outputURL)
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
let payload = try Data(contentsOf: eventsURL)
|
|
69
|
+
let envelope = try JSONDecoder().decode(GestureEnvelope.self, from: payload)
|
|
70
|
+
|
|
71
|
+
if envelope.events.isEmpty {
|
|
72
|
+
try FileManager.default.copyItem(at: inputURL, to: outputURL)
|
|
73
|
+
return
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
let asset = AVURLAsset(url: inputURL)
|
|
77
|
+
guard let sourceVideoTrack = asset.tracks(withMediaType: .video).first else {
|
|
78
|
+
throw OverlayError.missingVideoTrack
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
let composition = AVMutableComposition()
|
|
82
|
+
guard let compositionVideoTrack = composition.addMutableTrack(
|
|
83
|
+
withMediaType: .video,
|
|
84
|
+
preferredTrackID: kCMPersistentTrackID_Invalid
|
|
85
|
+
) else {
|
|
86
|
+
throw OverlayError.exportFailed("Failed to create composition video track.")
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
let fullRange = CMTimeRange(start: .zero, duration: asset.duration)
|
|
90
|
+
try compositionVideoTrack.insertTimeRange(fullRange, of: sourceVideoTrack, at: .zero)
|
|
91
|
+
|
|
92
|
+
if let sourceAudioTrack = asset.tracks(withMediaType: .audio).first,
|
|
93
|
+
let compositionAudioTrack = composition.addMutableTrack(
|
|
94
|
+
withMediaType: .audio,
|
|
95
|
+
preferredTrackID: kCMPersistentTrackID_Invalid
|
|
96
|
+
) {
|
|
97
|
+
try? compositionAudioTrack.insertTimeRange(fullRange, of: sourceAudioTrack, at: .zero)
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
let renderSize = resolvedRenderSize(for: sourceVideoTrack)
|
|
101
|
+
let videoComposition = AVMutableVideoComposition()
|
|
102
|
+
videoComposition.renderSize = renderSize
|
|
103
|
+
videoComposition.frameDuration = resolvedFrameDuration(for: sourceVideoTrack)
|
|
104
|
+
|
|
105
|
+
let instruction = AVMutableVideoCompositionInstruction()
|
|
106
|
+
instruction.timeRange = fullRange
|
|
107
|
+
let layerInstruction = AVMutableVideoCompositionLayerInstruction(assetTrack: compositionVideoTrack)
|
|
108
|
+
layerInstruction.setTransform(sourceVideoTrack.preferredTransform, at: .zero)
|
|
109
|
+
instruction.layerInstructions = [layerInstruction]
|
|
110
|
+
videoComposition.instructions = [instruction]
|
|
111
|
+
|
|
112
|
+
let parentLayer = CALayer()
|
|
113
|
+
parentLayer.frame = CGRect(origin: .zero, size: renderSize)
|
|
114
|
+
parentLayer.masksToBounds = true
|
|
115
|
+
|
|
116
|
+
let videoLayer = CALayer()
|
|
117
|
+
videoLayer.frame = parentLayer.frame
|
|
118
|
+
parentLayer.addSublayer(videoLayer)
|
|
119
|
+
|
|
120
|
+
let overlayLayer = CALayer()
|
|
121
|
+
overlayLayer.frame = parentLayer.frame
|
|
122
|
+
parentLayer.addSublayer(overlayLayer)
|
|
123
|
+
|
|
124
|
+
for event in envelope.events {
|
|
125
|
+
switch event.kind {
|
|
126
|
+
case "tap":
|
|
127
|
+
addTapLayer(event: event, renderSize: renderSize, to: overlayLayer)
|
|
128
|
+
case "longpress":
|
|
129
|
+
addLongPressLayer(event: event, renderSize: renderSize, to: overlayLayer)
|
|
130
|
+
case "swipe":
|
|
131
|
+
addSwipeLayers(event: event, renderSize: renderSize, to: overlayLayer)
|
|
132
|
+
case "scroll":
|
|
133
|
+
addScrollLayers(event: event, renderSize: renderSize, to: overlayLayer)
|
|
134
|
+
case "back-swipe":
|
|
135
|
+
addBackSwipeLayers(event: event, renderSize: renderSize, to: overlayLayer)
|
|
136
|
+
case "pinch":
|
|
137
|
+
addPinchLayers(event: event, renderSize: renderSize, to: overlayLayer)
|
|
138
|
+
default:
|
|
139
|
+
continue
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
videoComposition.animationTool = AVVideoCompositionCoreAnimationTool(
|
|
144
|
+
postProcessingAsVideoLayer: videoLayer,
|
|
145
|
+
in: parentLayer
|
|
146
|
+
)
|
|
147
|
+
|
|
148
|
+
guard let exporter = AVAssetExportSession(asset: composition, presetName: AVAssetExportPresetHighestQuality) else {
|
|
149
|
+
throw OverlayError.exportFailed("Failed to create export session.")
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
exporter.outputURL = outputURL
|
|
153
|
+
exporter.outputFileType = .mp4
|
|
154
|
+
exporter.videoComposition = videoComposition
|
|
155
|
+
exporter.shouldOptimizeForNetworkUse = true
|
|
156
|
+
|
|
157
|
+
let semaphore = DispatchSemaphore(value: 0)
|
|
158
|
+
exporter.exportAsynchronously {
|
|
159
|
+
semaphore.signal()
|
|
160
|
+
}
|
|
161
|
+
if semaphore.wait(timeout: .now() + 120) == .timedOut {
|
|
162
|
+
exporter.cancelExport()
|
|
163
|
+
throw OverlayError.exportFailed("Touch overlay export timed out.")
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
if exporter.status != .completed {
|
|
167
|
+
throw OverlayError.exportFailed(exporter.error?.localizedDescription ?? "Touch overlay export failed.")
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
func parseArguments(_ arguments: [String]) throws -> (inputPath: String, outputPath: String, eventsPath: String) {
|
|
172
|
+
var inputPath: String?
|
|
173
|
+
var outputPath: String?
|
|
174
|
+
var eventsPath: String?
|
|
175
|
+
var index = 0
|
|
176
|
+
|
|
177
|
+
while index < arguments.count {
|
|
178
|
+
let argument = arguments[index]
|
|
179
|
+
let nextIndex = index + 1
|
|
180
|
+
switch argument {
|
|
181
|
+
case "--input":
|
|
182
|
+
guard nextIndex < arguments.count else { throw OverlayError.invalidArgs("--input requires a value") }
|
|
183
|
+
inputPath = arguments[nextIndex]
|
|
184
|
+
index += 2
|
|
185
|
+
case "--output":
|
|
186
|
+
guard nextIndex < arguments.count else { throw OverlayError.invalidArgs("--output requires a value") }
|
|
187
|
+
outputPath = arguments[nextIndex]
|
|
188
|
+
index += 2
|
|
189
|
+
case "--events":
|
|
190
|
+
guard nextIndex < arguments.count else { throw OverlayError.invalidArgs("--events requires a value") }
|
|
191
|
+
eventsPath = arguments[nextIndex]
|
|
192
|
+
index += 2
|
|
193
|
+
default:
|
|
194
|
+
throw OverlayError.invalidArgs("Unknown argument: \(argument)")
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
guard let inputPath, let outputPath, let eventsPath else {
|
|
199
|
+
throw OverlayError.invalidArgs("Usage: recording-overlay.swift --input <video> --output <video> --events <json>")
|
|
200
|
+
}
|
|
201
|
+
return (inputPath, outputPath, eventsPath)
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
func resolvedRenderSize(for track: AVAssetTrack) -> CGSize {
|
|
205
|
+
let transformed = track.naturalSize.applying(track.preferredTransform)
|
|
206
|
+
return CGSize(width: abs(transformed.width), height: abs(transformed.height))
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
func resolvedFrameDuration(for track: AVAssetTrack) -> CMTime {
|
|
210
|
+
let minFrameDuration = track.minFrameDuration
|
|
211
|
+
if minFrameDuration.isValid && !minFrameDuration.isIndefinite && minFrameDuration.seconds > 0 {
|
|
212
|
+
return minFrameDuration
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
let nominalFrameRate = track.nominalFrameRate
|
|
216
|
+
if nominalFrameRate > 0 {
|
|
217
|
+
let timescale = Int32(max(1, round(nominalFrameRate)))
|
|
218
|
+
return CMTime(value: 1, timescale: timescale)
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
return CMTime(value: 1, timescale: 60)
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
func overlayPoint(event: GestureEvent, x: Double, y: Double, renderSize: CGSize) -> CGPoint {
|
|
225
|
+
let scaleX = scaledAxis(renderSize: renderSize.width, referenceSize: event.referenceWidth)
|
|
226
|
+
let scaleY = scaledAxis(renderSize: renderSize.height, referenceSize: event.referenceHeight)
|
|
227
|
+
let scaledX = x * scaleX
|
|
228
|
+
let scaledY = y * scaleY
|
|
229
|
+
let flippedY = max(0, Double(renderSize.height) - scaledY)
|
|
230
|
+
return CGPoint(x: scaledX, y: flippedY)
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
func scaledAxis(renderSize: CGFloat, referenceSize: Double?) -> Double {
|
|
234
|
+
guard let referenceSize, referenceSize > 0 else { return 1.0 }
|
|
235
|
+
return Double(renderSize) / referenceSize
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
func addTapLayer(event: GestureEvent, renderSize: CGSize, to overlayLayer: CALayer) {
|
|
239
|
+
let layer = makeTouchDotLayer(
|
|
240
|
+
center: overlayPoint(event: event, x: event.x, y: event.y, renderSize: renderSize),
|
|
241
|
+
renderSize: renderSize
|
|
242
|
+
)
|
|
243
|
+
overlayLayer.addSublayer(layer)
|
|
244
|
+
|
|
245
|
+
let opacity = CAKeyframeAnimation(keyPath: "opacity")
|
|
246
|
+
opacity.values = [0.0, 0.98, 0.98, 0.0]
|
|
247
|
+
opacity.keyTimes = [0.0, 0.08, 0.8, 1.0]
|
|
248
|
+
|
|
249
|
+
let scale = CAKeyframeAnimation(keyPath: "transform.scale")
|
|
250
|
+
scale.values = [0.84, 1.0, 1.0]
|
|
251
|
+
scale.keyTimes = [0.0, 0.22, 1.0]
|
|
252
|
+
|
|
253
|
+
let group = makeAnimationGroup(
|
|
254
|
+
animations: [opacity, scale],
|
|
255
|
+
duration: minimumTapVisibility,
|
|
256
|
+
beginTime: AVCoreAnimationBeginTimeAtZero + (event.tMs / 1000.0)
|
|
257
|
+
)
|
|
258
|
+
layer.add(group, forKey: "tap")
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
func addLongPressLayer(event: GestureEvent, renderSize: CGSize, to overlayLayer: CALayer) {
|
|
262
|
+
let duration = max(0.75, (event.durationMs ?? 800) / 1000.0)
|
|
263
|
+
let layer = makeTouchDotLayer(
|
|
264
|
+
center: overlayPoint(event: event, x: event.x, y: event.y, renderSize: renderSize),
|
|
265
|
+
renderSize: renderSize
|
|
266
|
+
)
|
|
267
|
+
overlayLayer.addSublayer(layer)
|
|
268
|
+
|
|
269
|
+
let opacity = CAKeyframeAnimation(keyPath: "opacity")
|
|
270
|
+
opacity.values = [0.0, 0.98, 0.98, 0.0]
|
|
271
|
+
opacity.keyTimes = [0.0, 0.08, 0.92, 1.0]
|
|
272
|
+
|
|
273
|
+
let scale = CAKeyframeAnimation(keyPath: "transform.scale")
|
|
274
|
+
scale.values = [0.84, 1.0, 1.0]
|
|
275
|
+
scale.keyTimes = [0.0, 0.15, 1.0]
|
|
276
|
+
|
|
277
|
+
let group = makeAnimationGroup(
|
|
278
|
+
animations: [opacity, scale],
|
|
279
|
+
duration: duration,
|
|
280
|
+
beginTime: AVCoreAnimationBeginTimeAtZero + (event.tMs / 1000.0)
|
|
281
|
+
)
|
|
282
|
+
layer.add(group, forKey: "longpress")
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
func addSwipeLayers(event: GestureEvent, renderSize: CGSize, to overlayLayer: CALayer) {
|
|
286
|
+
addTrailLayers(event: event, renderSize: renderSize, to: overlayLayer, style: .swipe)
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
func addScrollLayers(event: GestureEvent, renderSize: CGSize, to overlayLayer: CALayer) {
|
|
290
|
+
addTrailLayers(event: event, renderSize: renderSize, to: overlayLayer, style: .scroll)
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
func addBackSwipeLayers(event: GestureEvent, renderSize: CGSize, to overlayLayer: CALayer) {
|
|
294
|
+
addTrailLayers(event: event, renderSize: renderSize, to: overlayLayer, style: .backSwipe)
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
enum TrailStyle: Equatable {
|
|
298
|
+
case swipe
|
|
299
|
+
case scroll
|
|
300
|
+
case backSwipe
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
extension TrailStyle {
|
|
304
|
+
var tail: CFTimeInterval {
|
|
305
|
+
switch self {
|
|
306
|
+
case .swipe:
|
|
307
|
+
return swipeVisibilityTail
|
|
308
|
+
case .scroll:
|
|
309
|
+
return 0.08
|
|
310
|
+
case .backSwipe:
|
|
311
|
+
return 0.12
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
var lineWidth: CGFloat {
|
|
316
|
+
switch self {
|
|
317
|
+
case .swipe:
|
|
318
|
+
return 4
|
|
319
|
+
case .scroll:
|
|
320
|
+
return 5
|
|
321
|
+
case .backSwipe:
|
|
322
|
+
return 6
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
|
|
326
|
+
var color: CGColor {
|
|
327
|
+
switch self {
|
|
328
|
+
case .swipe:
|
|
329
|
+
return touchDotColor
|
|
330
|
+
case .scroll:
|
|
331
|
+
return NSColor(calibratedRed: 0.16, green: 0.74, blue: 0.88, alpha: 0.34).cgColor
|
|
332
|
+
case .backSwipe:
|
|
333
|
+
return NSColor(calibratedRed: 0.24, green: 0.69, blue: 1.0, alpha: 0.55).cgColor
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
var borderColor: CGColor {
|
|
338
|
+
switch self {
|
|
339
|
+
case .swipe:
|
|
340
|
+
return touchDotBorderColor
|
|
341
|
+
case .scroll:
|
|
342
|
+
return NSColor(calibratedRed: 0.92, green: 1.0, blue: 1.0, alpha: 0.48).cgColor
|
|
343
|
+
case .backSwipe:
|
|
344
|
+
return NSColor(calibratedRed: 0.94, green: 0.98, blue: 1.0, alpha: 0.8).cgColor
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
var trailOpacityValues: [NSNumber] {
|
|
349
|
+
switch self {
|
|
350
|
+
case .swipe:
|
|
351
|
+
return [0.0, 0.9, 0.35, 0.0]
|
|
352
|
+
case .scroll:
|
|
353
|
+
return [0.0, 0.5, 0.18, 0.0]
|
|
354
|
+
case .backSwipe:
|
|
355
|
+
return [0.0, 1.0, 0.45, 0.0]
|
|
356
|
+
}
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
var dotOpacityValues: [NSNumber] {
|
|
360
|
+
switch self {
|
|
361
|
+
case .swipe:
|
|
362
|
+
return [0.0, 1.0, 0.92, 0.0]
|
|
363
|
+
case .scroll:
|
|
364
|
+
return [0.0, 0.72, 0.4, 0.0]
|
|
365
|
+
case .backSwipe:
|
|
366
|
+
return [0.0, 1.0, 0.9, 0.0]
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
func makeAnimationGroup(
|
|
372
|
+
animations: [CAAnimation],
|
|
373
|
+
duration: CFTimeInterval,
|
|
374
|
+
beginTime: CFTimeInterval
|
|
375
|
+
) -> CAAnimationGroup {
|
|
376
|
+
let group = CAAnimationGroup()
|
|
377
|
+
group.animations = animations
|
|
378
|
+
group.duration = duration
|
|
379
|
+
group.beginTime = beginTime
|
|
380
|
+
group.fillMode = .both
|
|
381
|
+
group.isRemovedOnCompletion = false
|
|
382
|
+
return group
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
func addTrailLayers(
|
|
386
|
+
event: GestureEvent,
|
|
387
|
+
renderSize: CGSize,
|
|
388
|
+
to overlayLayer: CALayer,
|
|
389
|
+
style: TrailStyle
|
|
390
|
+
) {
|
|
391
|
+
guard let x2 = event.x2, let y2 = event.y2 else { return }
|
|
392
|
+
let startPoint = overlayPoint(event: event, x: event.x, y: event.y, renderSize: renderSize)
|
|
393
|
+
let endPoint = overlayPoint(event: event, x: x2, y: y2, renderSize: renderSize)
|
|
394
|
+
let duration = max(0.1, (event.durationMs ?? 250) / 1000.0)
|
|
395
|
+
let visibleDuration = max(minimumSwipeVisibility, duration + style.tail)
|
|
396
|
+
let beginTime = AVCoreAnimationBeginTimeAtZero + (event.tMs / 1000.0)
|
|
397
|
+
|
|
398
|
+
let pathLayer = CAShapeLayer()
|
|
399
|
+
pathLayer.frame = overlayLayer.bounds
|
|
400
|
+
pathLayer.strokeEnd = 1.0
|
|
401
|
+
pathLayer.path = {
|
|
402
|
+
let path = CGMutablePath()
|
|
403
|
+
path.move(to: startPoint)
|
|
404
|
+
path.addLine(to: endPoint)
|
|
405
|
+
return path
|
|
406
|
+
}()
|
|
407
|
+
pathLayer.strokeColor = style.color
|
|
408
|
+
pathLayer.lineWidth = style.lineWidth
|
|
409
|
+
pathLayer.lineCap = .round
|
|
410
|
+
pathLayer.fillColor = nil
|
|
411
|
+
pathLayer.opacity = 0
|
|
412
|
+
overlayLayer.addSublayer(pathLayer)
|
|
413
|
+
|
|
414
|
+
let stroke = CABasicAnimation(keyPath: "strokeEnd")
|
|
415
|
+
stroke.fromValue = 0.0
|
|
416
|
+
stroke.toValue = 1.0
|
|
417
|
+
|
|
418
|
+
let strokeOpacity = CAKeyframeAnimation(keyPath: "opacity")
|
|
419
|
+
strokeOpacity.values = style.trailOpacityValues
|
|
420
|
+
strokeOpacity.keyTimes = trailOpacityKeyTimes
|
|
421
|
+
|
|
422
|
+
let strokeGroup = makeAnimationGroup(
|
|
423
|
+
animations: [stroke, strokeOpacity],
|
|
424
|
+
duration: visibleDuration,
|
|
425
|
+
beginTime: beginTime
|
|
426
|
+
)
|
|
427
|
+
strokeGroup.timingFunction = CAMediaTimingFunction(name: .easeInEaseOut)
|
|
428
|
+
pathLayer.add(strokeGroup, forKey: "stroke")
|
|
429
|
+
|
|
430
|
+
let dotLayer = makeTouchDotLayer(center: startPoint, renderSize: renderSize)
|
|
431
|
+
dotLayer.backgroundColor = style.color
|
|
432
|
+
dotLayer.borderColor = style.borderColor
|
|
433
|
+
dotLayer.position = endPoint
|
|
434
|
+
overlayLayer.addSublayer(dotLayer)
|
|
435
|
+
|
|
436
|
+
let position = CABasicAnimation(keyPath: "position")
|
|
437
|
+
position.fromValue = NSValue(point: startPoint)
|
|
438
|
+
position.toValue = NSValue(point: endPoint)
|
|
439
|
+
position.duration = duration
|
|
440
|
+
|
|
441
|
+
let opacity = CAKeyframeAnimation(keyPath: "opacity")
|
|
442
|
+
opacity.values = style.dotOpacityValues
|
|
443
|
+
opacity.keyTimes = trailOpacityKeyTimes
|
|
444
|
+
|
|
445
|
+
let group = makeAnimationGroup(
|
|
446
|
+
animations: [position, opacity],
|
|
447
|
+
duration: visibleDuration,
|
|
448
|
+
beginTime: beginTime
|
|
449
|
+
)
|
|
450
|
+
group.timingFunction = CAMediaTimingFunction(name: .easeInEaseOut)
|
|
451
|
+
dotLayer.add(group, forKey: "swipe-dot")
|
|
452
|
+
|
|
453
|
+
if style == .backSwipe {
|
|
454
|
+
addBackSwipeEdgeHint(
|
|
455
|
+
event: event,
|
|
456
|
+
renderSize: renderSize,
|
|
457
|
+
beginTime: beginTime,
|
|
458
|
+
visibleDuration: visibleDuration,
|
|
459
|
+
to: overlayLayer
|
|
460
|
+
)
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
func addPinchDot(
|
|
465
|
+
overlayLayer: CALayer,
|
|
466
|
+
start: CGPoint,
|
|
467
|
+
end: CGPoint,
|
|
468
|
+
renderSize: CGSize,
|
|
469
|
+
beginTime: CFTimeInterval,
|
|
470
|
+
duration: CFTimeInterval
|
|
471
|
+
) {
|
|
472
|
+
let dotLayer = makeTouchDotLayer(center: start, renderSize: renderSize)
|
|
473
|
+
overlayLayer.addSublayer(dotLayer)
|
|
474
|
+
|
|
475
|
+
let position = CABasicAnimation(keyPath: "position")
|
|
476
|
+
position.fromValue = NSValue(point: start)
|
|
477
|
+
position.toValue = NSValue(point: end)
|
|
478
|
+
position.duration = duration
|
|
479
|
+
|
|
480
|
+
let opacity = CAKeyframeAnimation(keyPath: "opacity")
|
|
481
|
+
opacity.values = [0.0, 1.0, 1.0, 0.0]
|
|
482
|
+
opacity.keyTimes = [0.0, 0.1, 0.82, 1.0]
|
|
483
|
+
|
|
484
|
+
let group = makeAnimationGroup(
|
|
485
|
+
animations: [position, opacity],
|
|
486
|
+
duration: duration,
|
|
487
|
+
beginTime: beginTime
|
|
488
|
+
)
|
|
489
|
+
dotLayer.add(group, forKey: "pinch")
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
func addPinchLayers(event: GestureEvent, renderSize: CGSize, to overlayLayer: CALayer) {
|
|
493
|
+
let duration = max(minimumPinchVisibility, (event.durationMs ?? 280) / 1000.0)
|
|
494
|
+
let beginTime = AVCoreAnimationBeginTimeAtZero + (event.tMs / 1000.0)
|
|
495
|
+
let startOffset: CGFloat = 28
|
|
496
|
+
let scale = max(0.2, min(event.scale ?? 1.0, 3.0))
|
|
497
|
+
let endOffset = scale >= 1.0 ? startOffset * CGFloat(min(scale, 2.0)) : startOffset * CGFloat(max(scale, 0.5))
|
|
498
|
+
let startLeft = overlayPoint(event: event, x: event.x - Double(startOffset), y: event.y, renderSize: renderSize)
|
|
499
|
+
let startRight = overlayPoint(event: event, x: event.x + Double(startOffset), y: event.y, renderSize: renderSize)
|
|
500
|
+
let endLeft = overlayPoint(event: event, x: event.x - Double(endOffset), y: event.y, renderSize: renderSize)
|
|
501
|
+
let endRight = overlayPoint(event: event, x: event.x + Double(endOffset), y: event.y, renderSize: renderSize)
|
|
502
|
+
|
|
503
|
+
addPinchDot(
|
|
504
|
+
overlayLayer: overlayLayer,
|
|
505
|
+
start: startLeft,
|
|
506
|
+
end: endLeft,
|
|
507
|
+
renderSize: renderSize,
|
|
508
|
+
beginTime: beginTime,
|
|
509
|
+
duration: duration
|
|
510
|
+
)
|
|
511
|
+
addPinchDot(
|
|
512
|
+
overlayLayer: overlayLayer,
|
|
513
|
+
start: startRight,
|
|
514
|
+
end: endRight,
|
|
515
|
+
renderSize: renderSize,
|
|
516
|
+
beginTime: beginTime,
|
|
517
|
+
duration: duration
|
|
518
|
+
)
|
|
519
|
+
}
|
|
520
|
+
|
|
521
|
+
func makeTouchDotLayer(center: CGPoint, renderSize: CGSize) -> CALayer {
|
|
522
|
+
let dotRadius = resolvedTouchDotRadius(renderSize: renderSize)
|
|
523
|
+
let layer = CALayer()
|
|
524
|
+
layer.bounds = CGRect(x: 0, y: 0, width: dotRadius * 2, height: dotRadius * 2)
|
|
525
|
+
layer.position = center
|
|
526
|
+
layer.cornerRadius = dotRadius
|
|
527
|
+
layer.backgroundColor = touchDotColor
|
|
528
|
+
layer.borderWidth = 2
|
|
529
|
+
layer.borderColor = touchDotBorderColor
|
|
530
|
+
layer.shadowColor = NSColor(calibratedRed: 0.08, green: 0.20, blue: 0.36, alpha: 1.0).cgColor
|
|
531
|
+
layer.shadowOpacity = 0.18
|
|
532
|
+
layer.shadowRadius = 4
|
|
533
|
+
layer.opacity = 0
|
|
534
|
+
return layer
|
|
535
|
+
}
|
|
536
|
+
|
|
537
|
+
func resolvedTouchDotRadius(renderSize: CGSize) -> CGFloat {
|
|
538
|
+
let minDimension = min(renderSize.width, renderSize.height)
|
|
539
|
+
return max(18, min(40, minDimension * 0.035))
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
func addBackSwipeEdgeHint(
|
|
543
|
+
event: GestureEvent,
|
|
544
|
+
renderSize: CGSize,
|
|
545
|
+
beginTime: CFTimeInterval,
|
|
546
|
+
visibleDuration: CFTimeInterval,
|
|
547
|
+
to overlayLayer: CALayer
|
|
548
|
+
) {
|
|
549
|
+
let edge = (event.edge ?? "left").lowercased()
|
|
550
|
+
let hintLayer = CALayer()
|
|
551
|
+
let width: CGFloat = 10
|
|
552
|
+
let height: CGFloat = min(renderSize.height * 0.3, 320)
|
|
553
|
+
let y = (renderSize.height - height) / 2
|
|
554
|
+
let x: CGFloat = edge == "right" ? renderSize.width - width : 0
|
|
555
|
+
hintLayer.frame = CGRect(x: x, y: y, width: width, height: height)
|
|
556
|
+
hintLayer.backgroundColor = NSColor(calibratedRed: 0.24, green: 0.69, blue: 1.0, alpha: 0.22).cgColor
|
|
557
|
+
hintLayer.cornerRadius = width / 2
|
|
558
|
+
hintLayer.opacity = 0
|
|
559
|
+
overlayLayer.addSublayer(hintLayer)
|
|
560
|
+
|
|
561
|
+
let opacity = CAKeyframeAnimation(keyPath: "opacity")
|
|
562
|
+
opacity.values = [0.0, 0.9, 0.0]
|
|
563
|
+
opacity.keyTimes = [0.0, 0.2, 1.0]
|
|
564
|
+
|
|
565
|
+
let group = makeAnimationGroup(
|
|
566
|
+
animations: [opacity],
|
|
567
|
+
duration: visibleDuration,
|
|
568
|
+
beginTime: beginTime
|
|
569
|
+
)
|
|
570
|
+
hintLayer.add(group, forKey: "back-swipe-edge")
|
|
571
|
+
}
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
import AVFoundation
|
|
2
|
+
import Foundation
|
|
3
|
+
|
|
4
|
+
enum TrimError: Error, CustomStringConvertible {
|
|
5
|
+
case invalidArgs(String)
|
|
6
|
+
case invalidTrimRange
|
|
7
|
+
case missingVideoTrack
|
|
8
|
+
case exportFailed(String)
|
|
9
|
+
|
|
10
|
+
var description: String {
|
|
11
|
+
switch self {
|
|
12
|
+
case .invalidArgs(let message):
|
|
13
|
+
return message
|
|
14
|
+
case .invalidTrimRange:
|
|
15
|
+
return "Trim start must be before the end of the recording."
|
|
16
|
+
case .missingVideoTrack:
|
|
17
|
+
return "Input video does not contain a video track."
|
|
18
|
+
case .exportFailed(let message):
|
|
19
|
+
return message
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
do {
|
|
25
|
+
try run()
|
|
26
|
+
} catch {
|
|
27
|
+
fputs("recording-trim: \(error)\n", stderr)
|
|
28
|
+
exit(1)
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
func run() throws {
|
|
32
|
+
let arguments = Array(CommandLine.arguments.dropFirst())
|
|
33
|
+
let parsedArgs = try parseArguments(arguments)
|
|
34
|
+
let inputURL = URL(fileURLWithPath: parsedArgs.inputPath)
|
|
35
|
+
let outputURL = URL(fileURLWithPath: parsedArgs.outputPath)
|
|
36
|
+
|
|
37
|
+
if FileManager.default.fileExists(atPath: outputURL.path) {
|
|
38
|
+
try FileManager.default.removeItem(at: outputURL)
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
let asset = AVURLAsset(url: inputURL)
|
|
42
|
+
guard let sourceVideoTrack = asset.tracks(withMediaType: .video).first else {
|
|
43
|
+
throw TrimError.missingVideoTrack
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
let trimStart = CMTime(seconds: parsedArgs.trimStartMs / 1000.0, preferredTimescale: 600)
|
|
47
|
+
guard CMTimeCompare(trimStart, asset.duration) < 0 else {
|
|
48
|
+
throw TrimError.invalidTrimRange
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
let trimmedDuration = CMTimeSubtract(asset.duration, trimStart)
|
|
52
|
+
guard CMTimeCompare(trimmedDuration, .zero) > 0 else {
|
|
53
|
+
throw TrimError.invalidTrimRange
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
let composition = AVMutableComposition()
|
|
57
|
+
let trimmedRange = CMTimeRange(start: trimStart, duration: trimmedDuration)
|
|
58
|
+
|
|
59
|
+
guard let compositionVideoTrack = composition.addMutableTrack(
|
|
60
|
+
withMediaType: .video,
|
|
61
|
+
preferredTrackID: kCMPersistentTrackID_Invalid
|
|
62
|
+
) else {
|
|
63
|
+
throw TrimError.exportFailed("Failed to create composition video track.")
|
|
64
|
+
}
|
|
65
|
+
try compositionVideoTrack.insertTimeRange(trimmedRange, of: sourceVideoTrack, at: .zero)
|
|
66
|
+
compositionVideoTrack.preferredTransform = sourceVideoTrack.preferredTransform
|
|
67
|
+
|
|
68
|
+
if let sourceAudioTrack = asset.tracks(withMediaType: .audio).first,
|
|
69
|
+
let compositionAudioTrack = composition.addMutableTrack(
|
|
70
|
+
withMediaType: .audio,
|
|
71
|
+
preferredTrackID: kCMPersistentTrackID_Invalid
|
|
72
|
+
) {
|
|
73
|
+
try? compositionAudioTrack.insertTimeRange(trimmedRange, of: sourceAudioTrack, at: .zero)
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
let presetName = AVAssetExportSession.exportPresets(compatibleWith: composition)
|
|
77
|
+
.contains(AVAssetExportPresetPassthrough)
|
|
78
|
+
? AVAssetExportPresetPassthrough
|
|
79
|
+
: AVAssetExportPresetHighestQuality
|
|
80
|
+
guard let exporter = AVAssetExportSession(asset: composition, presetName: presetName) else {
|
|
81
|
+
throw TrimError.exportFailed("Failed to create export session.")
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
exporter.outputURL = outputURL
|
|
85
|
+
exporter.outputFileType = .mp4
|
|
86
|
+
exporter.shouldOptimizeForNetworkUse = true
|
|
87
|
+
|
|
88
|
+
let semaphore = DispatchSemaphore(value: 0)
|
|
89
|
+
exporter.exportAsynchronously {
|
|
90
|
+
semaphore.signal()
|
|
91
|
+
}
|
|
92
|
+
if semaphore.wait(timeout: .now() + 120) == .timedOut {
|
|
93
|
+
exporter.cancelExport()
|
|
94
|
+
throw TrimError.exportFailed("Trim export timed out.")
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
if exporter.status != .completed {
|
|
98
|
+
throw TrimError.exportFailed(exporter.error?.localizedDescription ?? "Trim export failed.")
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
func parseArguments(_ arguments: [String]) throws -> (inputPath: String, outputPath: String, trimStartMs: Double) {
|
|
103
|
+
var inputPath: String?
|
|
104
|
+
var outputPath: String?
|
|
105
|
+
var trimStartMs: Double?
|
|
106
|
+
var index = 0
|
|
107
|
+
|
|
108
|
+
while index < arguments.count {
|
|
109
|
+
let argument = arguments[index]
|
|
110
|
+
let nextIndex = index + 1
|
|
111
|
+
switch argument {
|
|
112
|
+
case "--input":
|
|
113
|
+
guard nextIndex < arguments.count else { throw TrimError.invalidArgs("--input requires a value") }
|
|
114
|
+
inputPath = arguments[nextIndex]
|
|
115
|
+
index += 2
|
|
116
|
+
case "--output":
|
|
117
|
+
guard nextIndex < arguments.count else { throw TrimError.invalidArgs("--output requires a value") }
|
|
118
|
+
outputPath = arguments[nextIndex]
|
|
119
|
+
index += 2
|
|
120
|
+
case "--trim-start-ms":
|
|
121
|
+
guard nextIndex < arguments.count else {
|
|
122
|
+
throw TrimError.invalidArgs("--trim-start-ms requires a value")
|
|
123
|
+
}
|
|
124
|
+
guard let parsed = Double(arguments[nextIndex]), parsed >= 0 else {
|
|
125
|
+
throw TrimError.invalidArgs("--trim-start-ms must be a non-negative number")
|
|
126
|
+
}
|
|
127
|
+
trimStartMs = parsed
|
|
128
|
+
index += 2
|
|
129
|
+
default:
|
|
130
|
+
throw TrimError.invalidArgs("Unknown argument: \(argument)")
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
guard let inputPath, let outputPath, let trimStartMs else {
|
|
135
|
+
throw TrimError.invalidArgs(
|
|
136
|
+
"Usage: recording-trim.swift --input <video> --output <video> --trim-start-ms <ms>"
|
|
137
|
+
)
|
|
138
|
+
}
|
|
139
|
+
return (inputPath, outputPath, trimStartMs)
|
|
140
|
+
}
|