@t3lnet/sceneforge 1.0.33 → 1.0.34

package/cli/commands/record-demo.js

@@ -274,15 +274,26 @@ export async function runRecordDemoCommand(argv) {
     if (startUrl) {
       await page.goto(startUrl, { waitUntil: "networkidle" });
     }
+
+    // Build context with optional step callback for CDP recording
+    const demoContext = {
+      page,
+      baseURL: baseUrl,
+      outputDir: outputPaths.outputDir,
+      assetBaseDir: assetRoot ? toAbsolute(rootDir, assetRoot) : undefined,
+      videoRecordingStartTime,
+    };
+
+    // Add step callback for frame-accurate step markers when using CDP recording
+    if (cdpRecorder) {
+      demoContext.onStepStart = (stepId, stepIndex) => {
+        cdpRecorder.markStep(stepId, stepIndex);
+      };
+    }
+
     result = await runDemo(
       definition,
-      {
-        page,
-        baseURL: baseUrl,
-        outputDir: outputPaths.outputDir,
-        assetBaseDir: assetRoot ? toAbsolute(rootDir, assetRoot) : undefined,
-        videoRecordingStartTime,
-      },
+      demoContext,
       {
         generateScripts: true,
         scriptOutputDir: path.join(outputPaths.outputDir, "scripts"),

package/cli/commands/split-video.js

@@ -97,7 +97,6 @@ async function splitDemo(demoName, paths, outputDimensions = null) {
   }
 
   console.log(`[split] Video: ${videoPath}`);
-  console.log(`[split] Steps: ${script.stepBoundaries.length}`);
 
   const videoDurationSec = await getMediaDuration(videoPath);
   const videoDurationMs = Math.round(videoDurationSec * 1000);
@@ -105,30 +104,71 @@ async function splitDemo(demoName, paths, outputDimensions = null) {
   const stepClipsDir = path.join(paths.videosDir, demoName);
   await fs.mkdir(stepClipsDir, { recursive: true });
 
-  for (const boundary of script.stepBoundaries) {
-    const isFirstStep = boundary.stepIndex === 0;
-    const startMs = isFirstStep ? 0 : boundary.videoStartMs;
-    if (startMs >= videoDurationMs) {
-      console.warn(`[split]   Skipping ${boundary.stepId}: start beyond video duration`);
-      continue;
+  // Check for CDP-generated step manifest (frame-accurate boundaries)
+  const cdpManifestPath = videoPath.replace(/\.[^.]+$/, '-steps.json');
+  let useCdpManifest = false;
+  let cdpStepBoundaries = [];
+
+  try {
+    const cdpManifestContent = await fs.readFile(cdpManifestPath, 'utf-8');
+    const cdpManifest = JSON.parse(cdpManifestContent);
+    if (cdpManifest.steps && cdpManifest.steps.length > 0) {
+      cdpStepBoundaries = cdpManifest.steps;
+      useCdpManifest = true;
+      console.log(`[split] Using CDP step manifest: ${cdpManifestPath}`);
+      console.log(`[split] Steps: ${cdpStepBoundaries.length} (frame-accurate)`);
+    }
+  } catch {
+    // CDP manifest not found or invalid, fall back to script boundaries
+  }
+
+  // Use CDP manifest boundaries if available, otherwise fall back to script
+  const stepBoundaries = useCdpManifest ? cdpStepBoundaries : script.stepBoundaries;
+
+  if (!useCdpManifest) {
+    console.log(`[split] Steps: ${stepBoundaries.length} (from script)`);
+  }
+
+  for (const boundary of stepBoundaries) {
+    // Handle both CDP manifest format and script format
+    const stepIndex = boundary.stepIndex;
+    const stepId = boundary.stepId;
+
+    let startSec, endSec;
+    if (useCdpManifest) {
+      // CDP manifest has video times in seconds
+      startSec = boundary.videoStartSec;
+      endSec = boundary.videoEndSec;
+    } else {
+      // Script has times in milliseconds, need conversion and clamping
+      const isFirstStep = stepIndex === 0;
+      const startMs = isFirstStep ? 0 : boundary.videoStartMs;
+      if (startMs >= videoDurationMs) {
+        console.warn(`[split]   Skipping ${stepId}: start beyond video duration`);
+        continue;
+      }
+      const clampedEndMs = Math.min(boundary.videoEndMs, videoDurationMs);
+      if (clampedEndMs <= startMs) {
+        console.warn(`[split]   Skipping ${stepId}: invalid duration after clamp`);
+        continue;
+      }
+      startSec = startMs / 1000;
+      endSec = clampedEndMs / 1000;
     }
-    const clampedEndMs = Math.min(boundary.videoEndMs, videoDurationMs);
-    if (clampedEndMs <= startMs) {
-      console.warn(`[split]   Skipping ${boundary.stepId}: invalid duration after clamp`);
+
+    const duration = endSec - startSec;
+    if (duration <= 0) {
+      console.warn(`[split]   Skipping ${stepId}: invalid duration ${duration.toFixed(2)}s`);
       continue;
     }
-    const startSec = startMs / 1000;
-    const duration = (clampedEndMs - startMs) / 1000;
-    const paddedIndex = String(boundary.stepIndex + 1).padStart(2, "0");
-    const safeStepId = sanitizeFileSegment(
-      boundary.stepId,
-      `step-${boundary.stepIndex + 1}`
-    );
+
+    const paddedIndex = String(stepIndex + 1).padStart(2, "0");
+    const safeStepId = sanitizeFileSegment(stepId, `step-${stepIndex + 1}`);
     const outputFileName = `step_${paddedIndex}_${safeStepId}.mp4`;
     const outputPath = path.join(stepClipsDir, outputFileName);
 
     console.log(
-      `[split]   ${paddedIndex}. ${boundary.stepId}: ${startSec.toFixed(2)}s - ${(startSec + duration).toFixed(2)}s (${duration.toFixed(2)}s)`
+      `[split]   ${paddedIndex}. ${stepId}: ${startSec.toFixed(2)}s - ${endSec.toFixed(2)}s (${duration.toFixed(2)}s)`
     );
 
     try {
@@ -149,7 +189,7 @@ async function splitDemo(demoName, paths, outputDimensions = null) {
         outputPath,
       ]);
     } catch (error) {
-      console.error(`[split] ✗ Failed to extract step ${boundary.stepId}:`, error);
+      console.error(`[split] ✗ Failed to extract step ${stepId}:`, error);
       throw error;
     }
   }
@@ -161,31 +201,41 @@ async function splitDemo(demoName, paths, outputDimensions = null) {
     generatedAt: new Date().toISOString(),
     sourceVideo: videoPath,
     sourceVideoDurationMs: videoDurationMs,
-    steps: script.stepBoundaries.map((boundary) => {
+    usedCdpManifest: useCdpManifest,
+    steps: stepBoundaries.map((boundary) => {
       const paddedIndex = String(boundary.stepIndex + 1).padStart(2, "0");
-      const isFirstStep = boundary.stepIndex === 0;
-      const splitStartMs = isFirstStep ? 0 : boundary.videoStartMs;
-      const clampedEndMs = Math.min(boundary.videoEndMs, videoDurationMs);
       const safeStepId = sanitizeFileSegment(
         boundary.stepId,
         `step-${boundary.stepIndex + 1}`
       );
+
+      let splitStartMs, splitEndMs;
+      if (useCdpManifest) {
+        // CDP manifest has times in seconds
+        splitStartMs = Math.round(boundary.videoStartSec * 1000);
+        splitEndMs = Math.round(boundary.videoEndSec * 1000);
+      } else {
+        // Script has times in milliseconds
+        const isFirstStep = boundary.stepIndex === 0;
+        splitStartMs = isFirstStep ? 0 : boundary.videoStartMs;
+        splitEndMs = Math.min(boundary.videoEndMs, videoDurationMs);
+      }
+
       return {
         stepId: boundary.stepId,
         safeStepId,
         stepIndex: boundary.stepIndex,
         videoFile: path.join(stepClipsDir, `step_${paddedIndex}_${safeStepId}.mp4`),
         splitStartMs,
-        originalStartMs: boundary.videoStartMs,
-        originalEndMs: boundary.videoEndMs,
-        durationMs: Math.max(0, clampedEndMs - splitStartMs),
+        splitEndMs,
+        durationMs: Math.max(0, splitEndMs - splitStartMs),
       };
     }),
   };
 
   await fs.writeFile(manifestPath, JSON.stringify(manifest, null, 2));
 
-  console.log(`\n[split] ✓ Split into ${script.stepBoundaries.length} clips`);
+  console.log(`\n[split] ✓ Split into ${stepBoundaries.length} clips`);
   console.log(`[split]   Output: ${stepClipsDir}`);
   console.log(`[split]   Manifest: ${manifestPath}`);
 }

package/cli/utils/cdp-recorder.js

@@ -60,6 +60,11 @@ export function createCDPRecorder(page, options) {
   let nextFrameNumber = 0;
   let recordingDuration = 0; // Duration in ms from start to stop
 
+  // Step tracking - maps frames to steps for accurate splitting
+  let currentStepId = null;
+  let currentStepIndex = -1;
+  let stepMarkers = []; // Array of { stepId, stepIndex, startFrame, endFrame }
+
   const frameHandler = (params) => {
     if (!isRecording) return;
 
@@ -95,6 +100,10 @@ export function createCDPRecorder(page, options) {
     const buffer = Buffer.from(data, "base64");
 
     // Queue frame writes to ensure sequential completion
+    // Capture current step info at time of frame capture (before async write)
+    const frameStepId = currentStepId;
+    const frameStepIndex = currentStepIndex;
+
     writeQueue = writeQueue.then(async () => {
       try {
         await fs.writeFile(framePath, buffer);
@@ -102,6 +111,8 @@ export function createCDPRecorder(page, options) {
           frameNumber,
           timestamp,
           path: framePath,
+          stepId: frameStepId,
+          stepIndex: frameStepIndex,
         });
       } catch (err) {
         console.warn(`[cdp-recorder] Failed to write frame ${frameNumber}:`, err.message);
@@ -161,13 +172,46 @@ export function createCDPRecorder(page, options) {
       console.log(`[cdp-recorder] First frame received after ${Date.now() - startWait}ms`);
     },
 
+    /**
+     * Mark the start of a new step. Call this when each step begins.
+     * This creates frame-accurate step boundaries for splitting.
+     * @param {string} stepId - The step identifier
+     * @param {number} stepIndex - The step index (0-based)
+     */
+    markStep(stepId, stepIndex) {
+      // Close out previous step marker if one exists
+      if (currentStepId !== null && stepMarkers.length > 0) {
+        const lastMarker = stepMarkers[stepMarkers.length - 1];
+        // End frame will be set to the frame before this one (nextFrameNumber - 1)
+        // But we'll finalize this when we have actual frames
+        lastMarker.endFrameNumber = nextFrameNumber - 1;
+      }
+
+      // Start new step marker
+      currentStepId = stepId;
+      currentStepIndex = stepIndex;
+      stepMarkers.push({
+        stepId,
+        stepIndex,
+        startFrameNumber: nextFrameNumber, // Will be the next frame captured
+        endFrameNumber: null, // Will be set when next step starts or recording ends
+      });
+    },
+
+    /**
+     * Get current step markers (for debugging/logging)
+     */
+    getStepMarkers() {
+      return [...stepMarkers];
+    },
+
     /**
      * Stop recording and return frame information
-     * @returns {Promise<{frameCount: number, duration: number, framesDir: string}>}
+     * @returns {Promise<{frameCount: number, duration: number, framesDir: string, stepBoundaries: Array}>}
      */
     async stop() {
       if (!isRecording || !cdpSession) {
-        return { frameCount: 0, duration: 0, framesDir };
+        return { frameCount: 0, duration: 0, framesDir, stepBoundaries: [] };
       }
 
       isRecording = false;
@@ -184,6 +228,38 @@ export function createCDPRecorder(page, options) {
       // Wait for all pending frame writes to complete
       await writeQueue;
 
+      // Finalize the last step marker
+      if (stepMarkers.length > 0) {
+        const lastMarker = stepMarkers[stepMarkers.length - 1];
+        if (lastMarker.endFrameNumber === null) {
+          lastMarker.endFrameNumber = frameTimestamps.length - 1;
+        }
+      }
+
+      // Build frame-accurate step boundaries from the actual captured frames
+      // This uses the step info attached to each frame for precise boundaries
+      const stepBoundaries = stepMarkers.map((marker) => {
+        // Find actual frames that belong to this step
+        const stepFrames = frameTimestamps.filter(f => f.stepIndex === marker.stepIndex);
+        if (stepFrames.length === 0) {
+          return null;
+        }
+
+        const firstFrame = stepFrames[0];
+        const lastFrame = stepFrames[stepFrames.length - 1];
+
+        return {
+          stepId: marker.stepId,
+          stepIndex: marker.stepIndex,
+          startFrameNumber: firstFrame.frameNumber,
+          endFrameNumber: lastFrame.frameNumber,
+          frameCount: stepFrames.length,
+          // Time-based boundaries derived from frame timestamps
+          startTimeSec: firstFrame.timestamp,
+          endTimeSec: lastFrame.timestamp,
+        };
+      }).filter(Boolean);
+
       const delayToFirstFrame = firstFrameTime ? firstFrameTime - startTime : 0;
       const actualFps = frameTimestamps.length > 0 && recordingDuration > 0
         ? frameTimestamps.length / (recordingDuration / 1000)
@@ -195,6 +271,9 @@ export function createCDPRecorder(page, options) {
       if (delayToFirstFrame > 100) {
         console.log(`[cdp-recorder] First frame delay: ${delayToFirstFrame}ms`);
       }
+      if (stepBoundaries.length > 0) {
+        console.log(`[cdp-recorder] Tracked ${stepBoundaries.length} steps with frame-accurate boundaries`);
+      }
 
       return {
         frameCount: frameTimestamps.length,
@@ -204,6 +283,7 @@ export function createCDPRecorder(page, options) {
         firstFrameTime, // Wall clock time when first frame was captured
         startTime,      // Wall clock time when recording was started
         actualFps,      // Actual capture framerate for real-time playback
+        stepBoundaries, // Frame-accurate step boundaries
       };
     },
 
@@ -307,6 +387,58 @@ export function createCDPRecorder(page, options) {
 
       console.log(`[cdp-recorder] Video assembled: ${outputPath}`);
 
+      // Build frame-to-video-time mapping
+      // Each frame's video start time = sum of all previous frame durations
+      let cumulativeTime = 0;
+      const frameVideoTimes = framesWithDuration.map((frame) => {
+        const videoStartTime = cumulativeTime;
+        cumulativeTime += frame.duration;
+        return {
+          frameNumber: frame.frameNumber,
+          stepId: frame.stepId,
+          stepIndex: frame.stepIndex,
+          videoStartTimeSec: videoStartTime,
+          videoEndTimeSec: cumulativeTime,
+          duration: frame.duration,
+        };
+      });
+
+      // Calculate step boundaries in video time
+      const videoStepBoundaries = [];
+      if (stepMarkers.length > 0) {
+        for (const marker of stepMarkers) {
+          const stepFrames = frameVideoTimes.filter(f => f.stepIndex === marker.stepIndex);
+          if (stepFrames.length === 0) continue;
+
+          const firstFrame = stepFrames[0];
+          const lastFrame = stepFrames[stepFrames.length - 1];
+
+          videoStepBoundaries.push({
+            stepId: marker.stepId,
+            stepIndex: marker.stepIndex,
+            startFrameNumber: firstFrame.frameNumber,
+            endFrameNumber: lastFrame.frameNumber,
+            frameCount: stepFrames.length,
+            videoStartSec: firstFrame.videoStartTimeSec,
+            videoEndSec: lastFrame.videoEndTimeSec,
+            durationSec: lastFrame.videoEndTimeSec - firstFrame.videoStartTimeSec,
+          });
+        }
+
+        // Write step manifest for the split command
+        const manifestPath = outputPath.replace(/\.[^.]+$/, '-steps.json');
+        await fs.writeFile(manifestPath, JSON.stringify({
+          videoPath: outputPath,
+          totalDurationSec: totalDuration,
+          outputFps,
+          frameCount: framesWithDuration.length,
+          steps: videoStepBoundaries,
+          generatedAt: new Date().toISOString(),
+        }, null, 2));
+
+        console.log(`[cdp-recorder] Step manifest written: ${manifestPath}`);
+      }
+
       return outputPath;
     },
 

package/dist/index.cjs

@@ -2410,6 +2410,9 @@ async function runDemo(definition, context, options = {}) {
       if (scriptGenerator && stepIndex > 0) {
         await waitForStepReady(context.page, step);
       }
+      if (context.onStepStart) {
+        context.onStepStart(step.id, stepIndex);
+      }
       if (scriptGenerator) {
         scriptGenerator.startStep(step.id);
       }