@swmansion/argent 0.16.2-next.8 → 0.16.2-next.9

package/dist/tool-server.cjs

@@ -143036,6 +143036,9 @@ var screenRecordingSessionBlueprint = {
       frameStream: null,
       lastFrameStreamError: null,
       pumpTimer: null,
+      trimStatic: true,
+      framesWritten: 0,
+      trimmedAnyFrames: false,
       wallClockStartMs: null,
       wallClockEndMs: null,
       timeLimitSeconds: null,
@@ -143486,6 +143489,7 @@ var OUTPUT_FPS2 = 30;
 var FRAME_INTERVAL_MS = 1e3 / OUTPUT_FPS2;
 var MAX_CATCHUP_FRAMES = 5;
 var MAX_BUFFERED_BYTES = 32 * 1024 * 1024;
+var STATIC_GRACE_MS = 1e3;
 var STREAM_CONNECT_TIMEOUT_MS = 1e4;
 var FIRST_FRAME_TIMEOUT_MS = 1e4;
 var START_FAILFAST_GRACE_MS = 800;
@@ -143542,21 +143546,50 @@ function ffmpegArgs(opts) {
 function framesDue(startedAtMs, nowMs) {
   return Math.floor((nowMs - startedAtMs) * OUTPUT_FPS2 / 1e3);
 }
+function sameFrame(a, b) {
+  if (a === b) return true;
+  if (!a || !b) return false;
+  return a.equals(b);
+}
 function startPump(api, stream) {
   const child = api.captureProcess;
-  const startedAt = api.wallClockStartMs ?? Date.now();
-  let written = 0;
+  const trim = api.trimStatic;
+  api.framesWritten = 0;
+  api.trimmedAnyFrames = false;
+  let paceBaseMs = api.wallClockStartMs ?? Date.now();
+  let paceBaseFrames = 0;
+  let lastFrame = null;
+  let lastChangeMs = paceBaseMs;
+  let dead = false;
   api.pumpTimer = setInterval(() => {
     const stdin = child?.stdin;
     if (!stdin || !stdin.writable) return;
     const frame = stream.latest;
     if (!frame) return;
     if (stdin.writableLength > MAX_BUFFERED_BYTES) return;
-    const missing = Math.min(framesDue(startedAt, Date.now()) - written, MAX_CATCHUP_FRAMES);
+    const now = Date.now();
+    if (trim) {
+      if (!sameFrame(frame, lastFrame)) {
+        lastFrame = frame;
+        lastChangeMs = now;
+      }
+      if (now - lastChangeMs > STATIC_GRACE_MS) {
+        dead = true;
+        api.trimmedAnyFrames = true;
+        return;
+      }
+      if (dead) {
+        dead = false;
+        paceBaseMs = now;
+        paceBaseFrames = api.framesWritten;
+      }
+    }
+    const target = paceBaseFrames + framesDue(paceBaseMs, now);
+    const missing = Math.min(target - api.framesWritten, MAX_CATCHUP_FRAMES);
     for (let i = 0; i < missing; i++) {
       if (!stdin.writable) return;
       stdin.write(frame);
-      written++;
+      api.framesWritten++;
     }
   }, FRAME_INTERVAL_MS);
 }
@@ -143659,6 +143692,8 @@ async function startCaptureLocked(api, params) {
   api.outputFile = outputFile;
   api.logoFile = logoFile;
   api.watermarkSkipped = watermarkSkipped;
+  api.trimStatic = params.trimStatic;
+  api.framesWritten = 0;
   api.captureProcess = child;
   api.frameStream = stream;
   api.recordingActive = true;
@@ -143756,6 +143791,7 @@ async function stopCapture(api) {
   const outputFile = api.outputFile;
   const logoFile = api.logoFile;
   const startedAtMs = api.wallClockStartMs;
+  const trimStatic = api.trimStatic;
   const endedEarly = api.recordingTimedOut || api.recordingExitedUnexpectedly;
   const streamError = api.frameStream?.error ?? api.lastFrameStreamError ?? null;
   const watermarkSkipped = api.watermarkSkipped;
@@ -143793,8 +143829,16 @@ async function stopCapture(api) {
       warning = [warning, `The watermark was not applied (${watermarkSkipped}).`].filter(Boolean).join(" ");
     }
     const size = await statNonEmptyOutput(outputFile, "screen_recording_stop");
-    const durationMs = startedAtMs === null ? null : (api.wallClockEndMs ?? Date.now()) - startedAtMs;
-    return { outputFile, sizeBytes: size, durationMs, ...warning ? { warning } : {} };
+    const wallClockMs = startedAtMs === null ? null : (api.wallClockEndMs ?? Date.now()) - startedAtMs;
+    const durationMs = trimStatic ? Math.round(api.framesWritten / OUTPUT_FPS2 * 1e3) : wallClockMs;
+    const trimmedMs = trimStatic && wallClockMs !== null && api.trimmedAnyFrames ? Math.max(0, wallClockMs - durationMs) : void 0;
+    return {
+      outputFile,
+      sizeBytes: size,
+      durationMs,
+      ...trimmedMs !== void 0 ? { wallClockMs, trimmedMs } : {},
+      ...warning ? { warning } : {}
+    };
   } catch (err) {
     const empty2 = await import_fs17.promises.stat(outputFile).then((s) => s.size === 0).catch(() => false);
     if (empty2) await import_fs17.promises.rm(outputFile, { force: true }).catch(() => {
@@ -143809,6 +143853,8 @@ async function stopCapture(api) {
     api.outputFile = null;
     api.logoFile = null;
     api.watermarkSkipped = null;
+    api.framesWritten = 0;
+    api.trimmedAnyFrames = false;
     api.wallClockStartMs = null;
     api.wallClockEndMs = null;
     api.timeLimitSeconds = null;
@@ -143829,6 +143875,9 @@ var zodSchema51 = external_exports.object({
   udid: external_exports.string().describe("Target device id from `list-devices` (iOS Simulator UDID or Android serial)."),
   timeLimitSeconds: external_exports.number().int().min(1).max(MAX_TIME_LIMIT_SECONDS).optional().describe(
     `Auto-stop cap in seconds (default ${DEFAULT_TIME_LIMIT_SECONDS}, max ${MAX_TIME_LIMIT_SECONDS}). Set it to slightly more than the interaction you plan to capture.`
+  ),
+  trimStatic: external_exports.boolean().optional().describe(
+    "Default true. Collapse stretches where the screen does not change: the first second of each still stretch is kept, then unchanged frames are dropped until something moves again, so a long recording with brief activity comes back short instead of full of dead air. The returned durationMs is the trimmed length; wallClockMs/trimmedMs report what was removed. Set false to keep a faithful real-time recording."
   )
 });
 var capability26 = {
@@ -143839,7 +143888,8 @@ function createScreenRecordingStartTool(registry2) {
   return {
     id: "screen-recording-start",
     capability: capability26,
-    description: `Start recording the device screen to a video file (h264 mp4, constant 30fps at the device's native resolution).
+    description: `Start recording the device screen to a video file (h264 mp4, 30fps at the device's native resolution).
+By default stretches where the screen does not change are trimmed out (see trimStatic), so a long session with only brief activity comes back as a short clip instead of minutes of dead air.
 The recording keeps running across other tool calls (every result carries a reminder) until \`screen-recording-stop\` is called or timeLimitSeconds elapses \u2014 immediately after starting, set yourself a reminder/wakeup for the expected end of the recording so it is never left running.
 Use when the user wants a video of an interaction, animation, or app behavior \u2014 for a single still frame use \`screenshot\` instead.
 Returns { status: "recording", timeLimitSeconds, outputFile } \u2014 the video is retrieved later by \`screen-recording-stop\`, not by reading outputFile directly.
@@ -143887,7 +143937,8 @@ Fails if a recording is already running on the device, the device is not booted,
       return startCapture(api, {
         streamUrl,
         timeLimitSeconds,
-        watermark: isFeatureEnabled("video-watermark")
+        watermark: isFeatureEnabled("video-watermark"),
+        trimStatic: params.trimStatic ?? true
       });
     }
   };
@@ -143908,7 +143959,7 @@ var screenRecordingStopTool = {
   description: `Stop the screen recording started by \`screen-recording-start\` and retrieve the video: frame capture ends and ffmpeg finalizes the mp4.
 Also retrieves the video when the recording already ended on its own (time limit reached, capture process died) \u2014 call it even after the cap fired.
 Use when the interaction being captured is finished, or a tool-result note reminds you a recording is still running.
-Returns { video, durationMs, warning? }; video is a downloadable artifact materialized to a local path.
+Returns { video, durationMs, wallClockMs?, trimmedMs?, warning? }; video is a downloadable artifact materialized to a local path. When static-frame trimming removed dead air, durationMs is the trimmed video length and wallClockMs/trimmedMs report the real duration and how much was cut.
 Fails if no recording (running or finished-but-unretrieved) exists for the given udid.`,
   searchHint: "stop end finish screen recording video capture save retrieve",
   zodSchema: zodSchema52,
@@ -143923,6 +143974,8 @@ Fails if no recording (running or finished-but-unretrieved) exists for the given
     const artifacts = requireArtifacts(ctx);
     const video = await artifacts.register(stopped.outputFile, { mimeType: "video/mp4" });
     const result = { video, durationMs: stopped.durationMs };
+    if (stopped.wallClockMs !== void 0) result.wallClockMs = stopped.wallClockMs;
+    if (stopped.trimmedMs !== void 0) result.trimmedMs = stopped.trimmedMs;
     if (stopped.warning) result.warning = stopped.warning;
     return result;
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@swmansion/argent",
-  "version": "0.16.2-next.8",
+  "version": "0.16.2-next.9",
   "description": "MCP server for iOS Simulator and Android Emulator control",
   "license": "Apache-2.0",
   "repository": {

package/skills/argent-screen-recording/SKILL.md

@@ -14,7 +14,7 @@ One recording per device at a time; different devices can record concurrently. R
 
 ## 2. Critical: never leave a recording running
 
-A recording does not stop itself before its `timeLimitSeconds` cap, and a forgotten one wastes disk and returns a video full of dead air. Two safety nets exist — use both:
+A recording does not stop itself before its `timeLimitSeconds` cap, so a forgotten one keeps capturing until the cap fires — holding the recording session, wasting disk, and delaying the video you are waiting on (and with `trimStatic: false` it comes back padded with dead air). Two safety nets exist — use both:
 
 1. **Set yourself a reminder the moment the recording starts.** You know the expected capture length (the interaction you are about to drive). Immediately after `screen-recording-start` returns, schedule a wake-up for that expected end time using whatever your harness provides — a built-in reminder/wakeup or scheduled-task tool if you have one, otherwise a background shell running `sleep <expected-seconds>` whose completion notification pulls you back. When it fires, call `screen-recording-stop`. Do not rely on remembering.
 2. **Read the tool-result notes.** While a recording is running, every argent tool result carries a `NOTE:` reminding you it is still going and how to stop it. If the note says the recording already ended (time limit hit), still call `screen-recording-stop` — that is what hands you the file.
@@ -27,15 +27,17 @@ A recording does not stop itself before its `timeLimitSeconds` cap, and a forgot
 2. Call `screen-recording-start` with `udid` and a `timeLimitSeconds` slightly above the expected interaction length (default 180, max 600).
 3. Set the end-of-recording reminder described in §2 — this step is not optional.
 4. Drive the interaction to capture: gestures, navigation, typing (`argent-device-interact`). Prefer `run-sequence` for tight multi-step interactions so tool-call latency does not pad the video.
-5. Call `screen-recording-stop` with the same `udid`. It returns `{ video, durationMs, warning? }`; `video` is an artifact — use its `hostPath` locally or download it via the artifacts endpoint. The video is already final when stop returns (the watermark is stamped during capture, not in a second pass), so stop takes well under a second.
+5. Call `screen-recording-stop` with the same `udid`. It returns `{ video, durationMs, wallClockMs?, trimmedMs?, warning? }`; `video` is an artifact — use its `hostPath` locally or download it via the artifacts endpoint. The video is already final when stop returns (the watermark is stamped during capture, not in a second pass), so stop takes well under a second.
 6. Check `warning`: it reports cap-triggered stops, early encoder exits, a dropped frame stream, and possibly-truncated containers. Verify the file plays (or at least has a sane size) before presenting it to the user.
 
+**Static-frame trimming (on by default).** Stretches where the screen does not change are collapsed: the first second of each still stretch is kept so pauses read naturally, then unchanged frames are dropped until something moves again (a change of even a couple of pixels counts). So you can leave a recording running across slow steps, waits, or thinking time without padding the clip with dead air — a 40-second session with 5 seconds of real activity comes back as a ~5-7 second video. When trimming removed anything, stop also returns `wallClockMs` (real elapsed time) and `trimmedMs` (how much was cut); `durationMs` is always the length of the video you actually get. Pass `trimStatic: false` to `screen-recording-start` when you want a faithful real-time recording (e.g. to measure how long something took on screen).
+
 ---
 
 ## 4. Platform notes and limits
 
 - **What can be recorded**: anything simulator-server drives — iOS simulators, Android emulators, and physical Android devices. The only length limit is `timeLimitSeconds` (max 600).
-- **The timeline is wall-clock accurate**: a device only emits a frame when its screen changes, so captured frames are re-paced onto a steady 30 fps timeline. A recording of a completely still screen is still a full-length video (and compresses to almost nothing), and `durationMs` matches the time you actually recorded.
+- **The timeline is paced to a steady 30 fps**: a device only emits a frame when its screen changes, so captured frames are re-paced onto a fixed timeline rather than bunching up. With static-frame trimming off (`trimStatic: false`) that timeline is wall-clock accurate — a completely still screen still comes back as a full-length video (compressing to almost nothing) and `durationMs` matches the time you actually recorded. With trimming on (the default, see §3) still stretches past the grace window are collapsed, so `durationMs` is the trimmed video length and `wallClockMs` carries the real elapsed time.
 - **Android**: records at the device's native resolution; secure screens (DRM, some password fields) come out black.
 - **Unsupported**: tvOS simulators, physical iPhones, Chromium apps, Vega/Fire TV, and remote (`remote:`-prefixed) simulators — none of them expose a readable frame stream. For a single still frame use `screenshot`; for a replayable interaction script use `argent-create-flow` instead of a video.
 - **ffmpeg is required**: it is the encoder, so `screen-recording-start` fails up front with an install hint if it is missing (`brew install ffmpeg`). It is resolved from `PATH` plus the usual Homebrew prefixes.