autokap 1.0.8 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/skill/OPCODE-REFERENCE.md +29 -1
- package/assets/skill/SKILL.md +2 -1
- package/dist/auth-capture.js +35 -2
- package/dist/billing-operation-logging.d.ts +4 -3
- package/dist/billing-operation-logging.js +3 -2
- package/dist/browser.d.ts +10 -10
- package/dist/browser.js +32 -28
- package/dist/capture-encryption.d.ts +3 -1
- package/dist/capture-encryption.js +21 -6
- package/dist/capture-strategy.js +3 -2
- package/dist/cli-config.d.ts +2 -1
- package/dist/cli-config.js +51 -2
- package/dist/cli-contract.d.ts +5 -1
- package/dist/cli-contract.js +7 -1
- package/dist/cli-runner-local.js +16 -3
- package/dist/cli-runner.js +165 -18
- package/dist/cli.js +25 -19
- package/dist/clip-begin-frame-recorder.d.ts +44 -0
- package/dist/clip-begin-frame-recorder.js +250 -0
- package/dist/clip-capture-backend.d.ts +25 -0
- package/dist/clip-capture-backend.js +189 -0
- package/dist/clip-capture-loop.d.ts +61 -0
- package/dist/clip-capture-loop.js +111 -0
- package/dist/clip-frame-recorder.d.ts +63 -0
- package/dist/clip-frame-recorder.js +305 -0
- package/dist/clip-postprocess.d.ts +31 -2
- package/dist/clip-postprocess.js +174 -57
- package/dist/clip-runtime.d.ts +18 -0
- package/dist/clip-runtime.js +67 -0
- package/dist/clip-scale.d.ts +10 -0
- package/dist/clip-scale.js +21 -0
- package/dist/clip-screencast-recorder.d.ts +42 -0
- package/dist/clip-screencast-recorder.js +242 -0
- package/dist/clip-sidecar.d.ts +54 -0
- package/dist/clip-sidecar.js +208 -0
- package/dist/cost-logging.d.ts +1 -1
- package/dist/env-validation.js +38 -4
- package/dist/execution-schema.d.ts +690 -360
- package/dist/execution-schema.js +98 -42
- package/dist/execution-types.d.ts +53 -3
- package/dist/execution-types.js +2 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.js +1 -0
- package/dist/llm-healer.d.ts +2 -10
- package/dist/llm-healer.js +109 -62
- package/dist/llm-provider.js +3 -0
- package/dist/opcode-actions.js +13 -0
- package/dist/opcode-runner.js +21 -12
- package/dist/program-signing.d.ts +1094 -0
- package/dist/program-signing.js +140 -0
- package/dist/provider-config.d.ts +5 -0
- package/dist/provider-config.js +28 -1
- package/dist/recovery-chain.js +40 -16
- package/dist/server-credit-usage.d.ts +1 -1
- package/dist/types.d.ts +8 -2
- package/dist/web-playwright-local.d.ts +31 -1
- package/dist/web-playwright-local.js +207 -37
- package/package.json +12 -2
|
@@ -0,0 +1,305 @@
|
|
|
1
|
+
import fs from 'node:fs/promises';
|
|
2
|
+
import os from 'node:os';
|
|
3
|
+
import path from 'node:path';
|
|
4
|
+
import { postProcessClipFrames } from './clip-postprocess.js';
|
|
5
|
+
import { resolvePhysicalCaptureSize } from './clip-scale.js';
|
|
6
|
+
import { logger } from './logger.js';
|
|
7
|
+
const DEFAULT_CLIP_SOURCE_FPS = 30;
|
|
8
|
+
const DEFAULT_CLIP_OUTPUT_FPS = 30;
|
|
9
|
+
const INTERMEDIATE_FRAME_FORMAT = 'jpeg';
|
|
10
|
+
const INTERMEDIATE_FRAME_QUALITY = 80;
|
|
11
|
+
const MAX_PENDING_FRAME_WRITES = 4;
|
|
12
|
+
export function buildCdpViewportClip(viewport, effectiveScale, scrollOffset) {
|
|
13
|
+
return {
|
|
14
|
+
x: Math.max(0, scrollOffset.x),
|
|
15
|
+
y: Math.max(0, scrollOffset.y),
|
|
16
|
+
width: Math.max(1, Math.round(viewport.width)),
|
|
17
|
+
height: Math.max(1, Math.round(viewport.height)),
|
|
18
|
+
scale: Math.max(0.1, effectiveScale),
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
export class FrameSequenceClipRecorder {
|
|
22
|
+
page;
|
|
23
|
+
options;
|
|
24
|
+
sourceFps;
|
|
25
|
+
outputFps;
|
|
26
|
+
workingDirPromise;
|
|
27
|
+
startedAt = Date.now();
|
|
28
|
+
frames = [];
|
|
29
|
+
stopRequested = false;
|
|
30
|
+
loopPromise = null;
|
|
31
|
+
finalResult = null;
|
|
32
|
+
frameDimensions = null;
|
|
33
|
+
lastCaptureError = null;
|
|
34
|
+
pendingWrites = new Set();
|
|
35
|
+
pendingWriteError = null;
|
|
36
|
+
cdpSession = null;
|
|
37
|
+
cdpSessionPromise = null;
|
|
38
|
+
cdpCaptureDisabled = false;
|
|
39
|
+
constructor(page, options) {
|
|
40
|
+
this.page = page;
|
|
41
|
+
this.options = options;
|
|
42
|
+
this.sourceFps = Math.max(1, Math.round(options.sourceFps ?? DEFAULT_CLIP_SOURCE_FPS));
|
|
43
|
+
this.outputFps = Math.max(1, Math.round(options.outputFps ?? DEFAULT_CLIP_OUTPUT_FPS));
|
|
44
|
+
this.workingDirPromise = options.baseDir
|
|
45
|
+
? Promise.resolve(options.baseDir)
|
|
46
|
+
: fs.mkdtemp(path.join(os.tmpdir(), 'autokap-clip-frames-'));
|
|
47
|
+
this.frameDimensions = resolvePhysicalCaptureSize(options.viewport, options.effectiveScale)
|
|
48
|
+
?? {
|
|
49
|
+
width: Math.max(2, Math.round(options.viewport.width)) & ~1,
|
|
50
|
+
height: Math.max(2, Math.round(options.viewport.height)) & ~1,
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
async start() {
|
|
54
|
+
const workingDir = await this.workingDirPromise;
|
|
55
|
+
await fs.mkdir(path.join(workingDir, 'frames'), { recursive: true });
|
|
56
|
+
await this.captureFrame(Date.now(), false);
|
|
57
|
+
this.loopPromise = this.runCaptureLoop();
|
|
58
|
+
}
|
|
59
|
+
async stop() {
|
|
60
|
+
if (this.finalResult) {
|
|
61
|
+
return this.finalResult;
|
|
62
|
+
}
|
|
63
|
+
this.stopRequested = true;
|
|
64
|
+
await this.loopPromise;
|
|
65
|
+
const stopAt = Date.now();
|
|
66
|
+
await this.captureFrame(stopAt, true);
|
|
67
|
+
await this.flushPendingWrites();
|
|
68
|
+
if (this.frames.length === 0) {
|
|
69
|
+
throw this.lastCaptureError ?? new Error('clip capture failed before any frame could be recorded');
|
|
70
|
+
}
|
|
71
|
+
if (this.pendingWriteError) {
|
|
72
|
+
throw this.pendingWriteError;
|
|
73
|
+
}
|
|
74
|
+
const workingDir = await this.workingDirPromise;
|
|
75
|
+
const outputDimensions = this.frameDimensions
|
|
76
|
+
?? resolvePhysicalCaptureSize(this.options.viewport, this.options.effectiveScale)
|
|
77
|
+
?? this.options.viewport;
|
|
78
|
+
const frameEntries = buildFrameEntries(this.frames, stopAt, this.sourceFps, this.outputFps);
|
|
79
|
+
const actualSourceFps = resolveActualSourceFps(this.frames, stopAt);
|
|
80
|
+
try {
|
|
81
|
+
const result = await postProcessClipFrames(frameEntries, workingDir, 'clip', {
|
|
82
|
+
...this.options.clipOptions,
|
|
83
|
+
mp4Width: outputDimensions.width,
|
|
84
|
+
mp4Height: outputDimensions.height,
|
|
85
|
+
});
|
|
86
|
+
if (!result.mp4Path || !result.thumbnailPath) {
|
|
87
|
+
throw new Error('clip packaging failed: missing MP4 or thumbnail output');
|
|
88
|
+
}
|
|
89
|
+
const [mp4Buffer, gifBuffer, thumbnailBuffer] = await Promise.all([
|
|
90
|
+
fs.readFile(result.mp4Path),
|
|
91
|
+
result.gifPath ? fs.readFile(result.gifPath) : Promise.resolve(null),
|
|
92
|
+
fs.readFile(result.thumbnailPath),
|
|
93
|
+
]);
|
|
94
|
+
const clipPackage = {
|
|
95
|
+
mp4: {
|
|
96
|
+
buffer: mp4Buffer,
|
|
97
|
+
mimeType: 'video/mp4',
|
|
98
|
+
},
|
|
99
|
+
...(gifBuffer
|
|
100
|
+
? {
|
|
101
|
+
gif: {
|
|
102
|
+
buffer: gifBuffer,
|
|
103
|
+
mimeType: 'image/gif',
|
|
104
|
+
},
|
|
105
|
+
}
|
|
106
|
+
: {}),
|
|
107
|
+
thumbnail: {
|
|
108
|
+
buffer: thumbnailBuffer,
|
|
109
|
+
mimeType: 'image/png',
|
|
110
|
+
},
|
|
111
|
+
captureMethod: 'frame_sequence',
|
|
112
|
+
requestedScale: this.options.requestedScale,
|
|
113
|
+
effectiveScale: this.options.effectiveScale,
|
|
114
|
+
sourceFps: this.sourceFps,
|
|
115
|
+
actualSourceFps,
|
|
116
|
+
outputFps: this.outputFps,
|
|
117
|
+
frameCount: this.frames.length,
|
|
118
|
+
dimensions: outputDimensions,
|
|
119
|
+
};
|
|
120
|
+
this.finalResult = {
|
|
121
|
+
buffer: mp4Buffer,
|
|
122
|
+
mimeType: 'video/mp4',
|
|
123
|
+
durationMs: result.durationMs,
|
|
124
|
+
trimStartMs: 0,
|
|
125
|
+
clipPackage,
|
|
126
|
+
};
|
|
127
|
+
return this.finalResult;
|
|
128
|
+
}
|
|
129
|
+
finally {
|
|
130
|
+
await this.cdpSession?.detach().catch(() => undefined);
|
|
131
|
+
await fs.rm(workingDir, { recursive: true, force: true }).catch(() => undefined);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
async abort() {
|
|
135
|
+
this.stopRequested = true;
|
|
136
|
+
await this.loopPromise?.catch(() => undefined);
|
|
137
|
+
await this.cdpSession?.detach().catch(() => undefined);
|
|
138
|
+
const workingDir = await this.workingDirPromise;
|
|
139
|
+
await fs.rm(workingDir, { recursive: true, force: true }).catch(() => undefined);
|
|
140
|
+
}
|
|
141
|
+
async runCaptureLoop() {
|
|
142
|
+
const frameIntervalMs = Math.max(1, Math.round(1000 / this.sourceFps));
|
|
143
|
+
let nextCaptureAt = this.startedAt + frameIntervalMs;
|
|
144
|
+
while (!this.stopRequested) {
|
|
145
|
+
const waitMs = nextCaptureAt - Date.now();
|
|
146
|
+
if (waitMs > 0) {
|
|
147
|
+
await sleep(waitMs);
|
|
148
|
+
}
|
|
149
|
+
if (this.stopRequested) {
|
|
150
|
+
return;
|
|
151
|
+
}
|
|
152
|
+
await this.captureFrame(Date.now(), true);
|
|
153
|
+
nextCaptureAt += frameIntervalMs;
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
async captureFrame(capturedAt, allowFailure) {
|
|
157
|
+
try {
|
|
158
|
+
const workingDir = await this.workingDirPromise;
|
|
159
|
+
const frameIndex = this.frames.length;
|
|
160
|
+
const framePath = path.join(workingDir, 'frames', `frame-${String(frameIndex).padStart(6, '0')}.jpg`);
|
|
161
|
+
const frameBuffer = await this.captureFrameBuffer();
|
|
162
|
+
await this.queueFrameWrite(framePath, frameBuffer);
|
|
163
|
+
this.frames.push({ path: framePath, capturedAt });
|
|
164
|
+
}
|
|
165
|
+
catch (error) {
|
|
166
|
+
const captureError = error instanceof Error ? error : new Error(String(error));
|
|
167
|
+
this.lastCaptureError = captureError;
|
|
168
|
+
if (!allowFailure) {
|
|
169
|
+
throw captureError;
|
|
170
|
+
}
|
|
171
|
+
logger.debug(`[clip] frame capture skipped: ${captureError.message}`);
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
async captureFrameBuffer() {
|
|
175
|
+
const scrollOffset = await this.readScrollOffset();
|
|
176
|
+
const cdpBuffer = await this.captureFrameViaCdp(scrollOffset);
|
|
177
|
+
if (cdpBuffer) {
|
|
178
|
+
return cdpBuffer;
|
|
179
|
+
}
|
|
180
|
+
return this.page.screenshot({
|
|
181
|
+
type: INTERMEDIATE_FRAME_FORMAT,
|
|
182
|
+
quality: INTERMEDIATE_FRAME_QUALITY,
|
|
183
|
+
fullPage: false,
|
|
184
|
+
scale: 'device',
|
|
185
|
+
});
|
|
186
|
+
}
|
|
187
|
+
async readScrollOffset() {
|
|
188
|
+
try {
|
|
189
|
+
return await this.page.evaluate(() => ({
|
|
190
|
+
x: window.scrollX || window.pageXOffset || 0,
|
|
191
|
+
y: window.scrollY || window.pageYOffset || 0,
|
|
192
|
+
}));
|
|
193
|
+
}
|
|
194
|
+
catch {
|
|
195
|
+
return { x: 0, y: 0 };
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
async captureFrameViaCdp(scrollOffset) {
|
|
199
|
+
const session = await this.getCdpSession();
|
|
200
|
+
if (!session) {
|
|
201
|
+
return null;
|
|
202
|
+
}
|
|
203
|
+
try {
|
|
204
|
+
const screenshot = await session.send('Page.captureScreenshot', {
|
|
205
|
+
format: INTERMEDIATE_FRAME_FORMAT,
|
|
206
|
+
quality: INTERMEDIATE_FRAME_QUALITY,
|
|
207
|
+
optimizeForSpeed: true,
|
|
208
|
+
clip: buildCdpViewportClip(this.options.viewport, this.options.effectiveScale, scrollOffset),
|
|
209
|
+
});
|
|
210
|
+
return Buffer.from(screenshot.data, 'base64');
|
|
211
|
+
}
|
|
212
|
+
catch (error) {
|
|
213
|
+
this.disableCdpCapture(error);
|
|
214
|
+
return null;
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
async getCdpSession() {
|
|
218
|
+
if (this.cdpCaptureDisabled) {
|
|
219
|
+
return null;
|
|
220
|
+
}
|
|
221
|
+
if (this.cdpSession) {
|
|
222
|
+
return this.cdpSession;
|
|
223
|
+
}
|
|
224
|
+
if (!this.cdpSessionPromise) {
|
|
225
|
+
this.cdpSessionPromise = (async () => {
|
|
226
|
+
try {
|
|
227
|
+
const context = this.page.context();
|
|
228
|
+
if (typeof context.newCDPSession !== 'function') {
|
|
229
|
+
this.cdpCaptureDisabled = true;
|
|
230
|
+
return null;
|
|
231
|
+
}
|
|
232
|
+
this.cdpSession = await context.newCDPSession(this.page);
|
|
233
|
+
return this.cdpSession;
|
|
234
|
+
}
|
|
235
|
+
catch (error) {
|
|
236
|
+
this.disableCdpCapture(error);
|
|
237
|
+
return null;
|
|
238
|
+
}
|
|
239
|
+
})();
|
|
240
|
+
}
|
|
241
|
+
return this.cdpSessionPromise;
|
|
242
|
+
}
|
|
243
|
+
disableCdpCapture(error) {
|
|
244
|
+
if (!this.cdpCaptureDisabled) {
|
|
245
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
246
|
+
logger.debug(`[clip] CDP frame capture unavailable, falling back to Playwright screenshot: ${message}`);
|
|
247
|
+
}
|
|
248
|
+
this.cdpCaptureDisabled = true;
|
|
249
|
+
this.cdpSessionPromise = Promise.resolve(null);
|
|
250
|
+
}
|
|
251
|
+
async queueFrameWrite(framePath, frameBuffer) {
|
|
252
|
+
await this.waitForPendingWriteCapacity();
|
|
253
|
+
if (this.pendingWriteError) {
|
|
254
|
+
throw this.pendingWriteError;
|
|
255
|
+
}
|
|
256
|
+
let writePromise;
|
|
257
|
+
writePromise = fs.writeFile(framePath, frameBuffer)
|
|
258
|
+
.catch((error) => {
|
|
259
|
+
this.pendingWriteError = error instanceof Error ? error : new Error(String(error));
|
|
260
|
+
throw this.pendingWriteError;
|
|
261
|
+
})
|
|
262
|
+
.finally(() => {
|
|
263
|
+
this.pendingWrites.delete(writePromise);
|
|
264
|
+
});
|
|
265
|
+
this.pendingWrites.add(writePromise);
|
|
266
|
+
}
|
|
267
|
+
async waitForPendingWriteCapacity() {
|
|
268
|
+
while (this.pendingWrites.size >= MAX_PENDING_FRAME_WRITES) {
|
|
269
|
+
await Promise.race(this.pendingWrites);
|
|
270
|
+
if (this.pendingWriteError) {
|
|
271
|
+
throw this.pendingWriteError;
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
async flushPendingWrites() {
|
|
276
|
+
while (this.pendingWrites.size > 0) {
|
|
277
|
+
await Promise.allSettled(Array.from(this.pendingWrites));
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
function buildFrameEntries(frames, stopAt, sourceFps, outputFps) {
|
|
282
|
+
const entries = [];
|
|
283
|
+
const minFrameDurationMs = Math.max(1, Math.min(Math.round(1000 / sourceFps), Math.round(1000 / outputFps)));
|
|
284
|
+
for (let index = 0; index < frames.length; index += 1) {
|
|
285
|
+
const frame = frames[index];
|
|
286
|
+
const nextTimestamp = frames[index + 1]?.capturedAt ?? stopAt;
|
|
287
|
+
entries.push({
|
|
288
|
+
path: frame.path,
|
|
289
|
+
durationMs: Math.max(minFrameDurationMs, nextTimestamp - frame.capturedAt),
|
|
290
|
+
});
|
|
291
|
+
}
|
|
292
|
+
return entries;
|
|
293
|
+
}
|
|
294
|
+
function sleep(ms) {
|
|
295
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
296
|
+
}
|
|
297
|
+
function resolveActualSourceFps(frames, stopAt) {
|
|
298
|
+
if (frames.length <= 1) {
|
|
299
|
+
return frames.length;
|
|
300
|
+
}
|
|
301
|
+
const firstTimestamp = frames[0].capturedAt;
|
|
302
|
+
const durationSec = Math.max(0.001, (stopAt - firstTimestamp) / 1000);
|
|
303
|
+
return Math.round((frames.length / durationSec) * 100) / 100;
|
|
304
|
+
}
|
|
305
|
+
//# sourceMappingURL=clip-frame-recorder.js.map
|
|
@@ -15,11 +15,40 @@ export declare function convertToGif(webmPath: string, outputPath: string, opts?
|
|
|
15
15
|
/**
|
|
16
16
|
* Convert a WebM recording to an MP4 with web-optimized settings.
|
|
17
17
|
*/
|
|
18
|
-
export declare function convertToMp4(webmPath: string, outputPath: string
|
|
18
|
+
export declare function convertToMp4(webmPath: string, outputPath: string, opts?: {
|
|
19
|
+
width?: number;
|
|
20
|
+
height?: number;
|
|
21
|
+
trimStartSec?: number;
|
|
22
|
+
maxDurationSec?: number;
|
|
23
|
+
holdLastFrameSec?: number;
|
|
24
|
+
fps?: number;
|
|
25
|
+
}): Promise<void>;
|
|
19
26
|
/**
|
|
20
27
|
* Extract the first frame of a WebM as a PNG thumbnail.
|
|
21
28
|
*/
|
|
22
29
|
export declare function extractThumbnail(webmPath: string, outputPath: string): Promise<void>;
|
|
30
|
+
/**
|
|
31
|
+
* Assemble a sequence of JPEG frames into an H.264 MP4.
|
|
32
|
+
* Expects frames named `frame_000000.jpg`, `frame_000001.jpg`, ... in framesDir.
|
|
33
|
+
*
|
|
34
|
+
* When `frameOffsetsMs` is provided (one offset per frame, ms from first
|
|
35
|
+
* frame), uses ffmpeg's concat demuxer to preserve the actual capture timing
|
|
36
|
+
* (VFR). This matters when the compositor is CPU-bound on heavy React UIs:
|
|
37
|
+
* frames arrive in bursts during idle periods and gaps during paint-heavy
|
|
38
|
+
* periods. Encoding those at uniform CFR would stretch bursts and compress
|
|
39
|
+
* gaps — viewers see temporally-distorted playback even though the average
|
|
40
|
+
* rate is correct. VFR lets each frame hold the screen for its true wall-
|
|
41
|
+
* clock duration.
|
|
42
|
+
*
|
|
43
|
+
* When `frameOffsetsMs` is absent, falls back to CFR at `fps` (useful for
|
|
44
|
+
* uniform-cadence captures like the synthetic smoke tests).
|
|
45
|
+
*/
|
|
46
|
+
export declare function assembleMp4FromFrames(opts: {
|
|
47
|
+
framesDir: string;
|
|
48
|
+
outputPath: string;
|
|
49
|
+
fps: number;
|
|
50
|
+
frameOffsetsMs?: number[];
|
|
51
|
+
}): Promise<void>;
|
|
23
52
|
/**
|
|
24
53
|
* Trim a recording: skip dead frames at the start and cap duration.
|
|
25
54
|
*/
|
|
@@ -37,6 +66,6 @@ export interface ClipPostProcessResult {
|
|
|
37
66
|
}
|
|
38
67
|
/**
|
|
39
68
|
* Full post-processing pipeline for a single clip recording:
|
|
40
|
-
*
|
|
69
|
+
* encode a normalized master MP4 → derive GIF + thumbnail → measure.
|
|
41
70
|
*/
|
|
42
71
|
export declare function postProcessClipRecording(webmPath: string, outputDir: string, clipId: string, options?: ClipOptions): Promise<ClipPostProcessResult>;
|
package/dist/clip-postprocess.js
CHANGED
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
import { execFile } from 'node:child_process';
|
|
2
|
-
import {
|
|
2
|
+
import { copyFile, stat } from 'node:fs/promises';
|
|
3
3
|
import path from 'node:path';
|
|
4
4
|
import { promisify } from 'node:util';
|
|
5
5
|
const execFileAsync = promisify(execFile);
|
|
6
6
|
// ── Default values ──────────────────────────────────────────────────
|
|
7
|
-
const DEFAULT_GIF_FPS =
|
|
8
|
-
const DEFAULT_GIF_MAX_WIDTH =
|
|
7
|
+
const DEFAULT_GIF_FPS = 24;
|
|
8
|
+
const DEFAULT_GIF_MAX_WIDTH = 1440;
|
|
9
9
|
const DEFAULT_MAX_DURATION_SEC = 8;
|
|
10
10
|
const DEFAULT_TRIM_START_SEC = 0.3;
|
|
11
|
+
const DEFAULT_MP4_FPS = 30;
|
|
11
12
|
// ── ffmpeg detection ────────────────────────────────────────────────
|
|
12
13
|
const cachedBinaryPaths = {};
|
|
13
14
|
async function ensureBinaryAvailable(binaryName) {
|
|
@@ -48,44 +49,103 @@ export async function convertToGif(webmPath, outputPath, opts = {}) {
|
|
|
48
49
|
const fps = opts.fps ?? DEFAULT_GIF_FPS;
|
|
49
50
|
const maxWidth = opts.maxWidth ?? DEFAULT_GIF_MAX_WIDTH;
|
|
50
51
|
const loopFlag = (opts.loop ?? true) ? '0' : '-1';
|
|
51
|
-
const paletteDir = path.dirname(outputPath);
|
|
52
|
-
const palettePath = path.join(paletteDir, `_palette_${Date.now()}.png`);
|
|
53
52
|
const scaleFilter = `fps=${fps},scale=${maxWidth}:-1:flags=lanczos`;
|
|
54
|
-
//
|
|
53
|
+
// Single-pass GIF generation with per-frame palette (paletteuse=new=1):
|
|
54
|
+
// each frame gets its own optimal 256-color palette, re-computed from the
|
|
55
|
+
// pixels in that frame. This is ~3× better visually than a single global
|
|
56
|
+
// palette for screen recordings where UI state transitions produce very
|
|
57
|
+
// different color distributions (modal open/close, theme accents, etc.).
|
|
58
|
+
// sierra2_4a = error-diffusion dither (clean diagonal pattern vs bayer
|
|
59
|
+
// dots). We intentionally DON'T use diff_mode=rectangle because it leaves
|
|
60
|
+
// stale-palette pixels in unchanged regions when `new=1` is active.
|
|
61
|
+
const lavfi = `${scaleFilter},split[a][b];` +
|
|
62
|
+
`[a]palettegen=stats_mode=single:max_colors=256:reserve_transparent=0[p];` +
|
|
63
|
+
`[b][p]paletteuse=dither=sierra2_4a:new=1`;
|
|
55
64
|
await execFileAsync(ffmpeg, [
|
|
56
65
|
'-i', webmPath,
|
|
57
|
-
'-
|
|
58
|
-
'-
|
|
66
|
+
'-lavfi', lavfi,
|
|
67
|
+
'-loop', loopFlag,
|
|
68
|
+
'-y', outputPath,
|
|
59
69
|
]);
|
|
60
|
-
|
|
70
|
+
}
|
|
71
|
+
function resolveVideoDimension(value) {
|
|
72
|
+
return Number.isFinite(value) ? Math.max(2, Math.round(Number(value))) & ~1 : null;
|
|
73
|
+
}
|
|
74
|
+
function resolveMp4Bitrate(width, height) {
|
|
75
|
+
const effectiveWidth = width ?? 1440;
|
|
76
|
+
const effectiveHeight = height ?? 900;
|
|
77
|
+
const megapixels = Math.max(0.1, effectiveWidth * effectiveHeight / 1e6);
|
|
78
|
+
const targetMbps = Math.min(40, Math.max(6, Math.round(megapixels * 7)));
|
|
79
|
+
return {
|
|
80
|
+
target: `${targetMbps}M`,
|
|
81
|
+
maxrate: `${targetMbps}M`,
|
|
82
|
+
bufsize: `${targetMbps * 2}M`,
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
async function probeVideoDimensions(ffprobe, inputPath) {
|
|
61
86
|
try {
|
|
62
|
-
await execFileAsync(
|
|
63
|
-
'-
|
|
64
|
-
'-
|
|
65
|
-
'-
|
|
66
|
-
'-
|
|
67
|
-
|
|
87
|
+
const { stdout } = await execFileAsync(ffprobe, [
|
|
88
|
+
'-v', 'error',
|
|
89
|
+
'-select_streams', 'v:0',
|
|
90
|
+
'-show_entries', 'stream=width,height',
|
|
91
|
+
'-of', 'csv=p=0:s=,',
|
|
92
|
+
inputPath,
|
|
68
93
|
]);
|
|
94
|
+
const [w, h] = stdout.trim().split(',').map(Number);
|
|
95
|
+
if (!Number.isFinite(w) || !Number.isFinite(h))
|
|
96
|
+
return null;
|
|
97
|
+
return { width: w, height: h };
|
|
69
98
|
}
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
try {
|
|
73
|
-
await stat(palettePath);
|
|
74
|
-
await rm(palettePath, { force: true });
|
|
75
|
-
}
|
|
76
|
-
catch { /* ignore */ }
|
|
99
|
+
catch {
|
|
100
|
+
return null;
|
|
77
101
|
}
|
|
78
102
|
}
|
|
79
103
|
/**
|
|
80
104
|
* Convert a WebM recording to an MP4 with web-optimized settings.
|
|
81
105
|
*/
|
|
82
|
-
export async function convertToMp4(webmPath, outputPath) {
|
|
106
|
+
export async function convertToMp4(webmPath, outputPath, opts = {}) {
|
|
83
107
|
const ffmpeg = await ensureFfmpegAvailable();
|
|
108
|
+
const ffprobe = await ensureBinaryAvailable('ffprobe');
|
|
109
|
+
const width = resolveVideoDimension(opts.width);
|
|
110
|
+
const height = resolveVideoDimension(opts.height);
|
|
111
|
+
const fps = Math.max(1, Math.round(opts.fps ?? DEFAULT_MP4_FPS));
|
|
112
|
+
const trimStartSec = Math.max(0, opts.trimStartSec ?? 0);
|
|
113
|
+
const maxDurationSec = Math.max(0.1, opts.maxDurationSec ?? DEFAULT_MAX_DURATION_SEC);
|
|
114
|
+
const holdLastFrameSec = Math.min(Math.max(opts.holdLastFrameSec ?? 0, 0), 10);
|
|
115
|
+
const outputDurationSec = maxDurationSec + holdLastFrameSec;
|
|
116
|
+
const filters = [
|
|
117
|
+
`fps=${fps}`,
|
|
118
|
+
'setpts=PTS-STARTPTS',
|
|
119
|
+
];
|
|
120
|
+
if (holdLastFrameSec > 0) {
|
|
121
|
+
filters.push(`tpad=stop_mode=clone:stop_duration=${holdLastFrameSec}`);
|
|
122
|
+
}
|
|
123
|
+
if (width && height) {
|
|
124
|
+
const sourceDims = await probeVideoDimensions(ffprobe, webmPath);
|
|
125
|
+
const needsScale = !sourceDims
|
|
126
|
+
|| Math.abs(sourceDims.width - width) > 1
|
|
127
|
+
|| Math.abs(sourceDims.height - height) > 1;
|
|
128
|
+
if (needsScale) {
|
|
129
|
+
filters.push(`scale=${width}:${height}:flags=lanczos`);
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
const bitrate = resolveMp4Bitrate(width, height);
|
|
84
133
|
await execFileAsync(ffmpeg, [
|
|
134
|
+
...(trimStartSec > 0 ? ['-ss', String(trimStartSec)] : []),
|
|
85
135
|
'-i', webmPath,
|
|
136
|
+
'-t', String(outputDurationSec),
|
|
137
|
+
'-vf', filters.join(','),
|
|
86
138
|
'-c:v', 'libx264',
|
|
87
139
|
'-preset', 'slow',
|
|
88
|
-
'-crf', '
|
|
140
|
+
'-crf', '16',
|
|
141
|
+
'-b:v', bitrate.target,
|
|
142
|
+
'-maxrate', bitrate.maxrate,
|
|
143
|
+
'-bufsize', bitrate.bufsize,
|
|
144
|
+
'-g', String(fps * 2),
|
|
145
|
+
'-keyint_min', String(fps),
|
|
146
|
+
'-sc_threshold', '0',
|
|
147
|
+
'-profile:v', 'high',
|
|
148
|
+
'-tune', 'animation',
|
|
89
149
|
'-pix_fmt', 'yuv420p',
|
|
90
150
|
'-movflags', '+faststart',
|
|
91
151
|
'-an',
|
|
@@ -103,6 +163,70 @@ export async function extractThumbnail(webmPath, outputPath) {
|
|
|
103
163
|
'-y', outputPath,
|
|
104
164
|
]);
|
|
105
165
|
}
|
|
166
|
+
/**
|
|
167
|
+
* Assemble a sequence of JPEG frames into an H.264 MP4.
|
|
168
|
+
* Expects frames named `frame_000000.jpg`, `frame_000001.jpg`, ... in framesDir.
|
|
169
|
+
*
|
|
170
|
+
* When `frameOffsetsMs` is provided (one offset per frame, ms from first
|
|
171
|
+
* frame), uses ffmpeg's concat demuxer to preserve the actual capture timing
|
|
172
|
+
* (VFR). This matters when the compositor is CPU-bound on heavy React UIs:
|
|
173
|
+
* frames arrive in bursts during idle periods and gaps during paint-heavy
|
|
174
|
+
* periods. Encoding those at uniform CFR would stretch bursts and compress
|
|
175
|
+
* gaps — viewers see temporally-distorted playback even though the average
|
|
176
|
+
* rate is correct. VFR lets each frame hold the screen for its true wall-
|
|
177
|
+
* clock duration.
|
|
178
|
+
*
|
|
179
|
+
* When `frameOffsetsMs` is absent, falls back to CFR at `fps` (useful for
|
|
180
|
+
* uniform-cadence captures like the synthetic smoke tests).
|
|
181
|
+
*/
|
|
182
|
+
export async function assembleMp4FromFrames(opts) {
|
|
183
|
+
const ffmpeg = await ensureFfmpegAvailable();
|
|
184
|
+
const fps = Number.isFinite(opts.fps) && opts.fps >= 1 ? opts.fps : 25;
|
|
185
|
+
if (opts.frameOffsetsMs && opts.frameOffsetsMs.length >= 2) {
|
|
186
|
+
// VFR path: generate a concat file with per-frame durations.
|
|
187
|
+
const offsets = opts.frameOffsetsMs;
|
|
188
|
+
const lines = [];
|
|
189
|
+
for (let i = 0; i < offsets.length; i++) {
|
|
190
|
+
const durationSec = i < offsets.length - 1
|
|
191
|
+
? Math.max(0.001, (offsets[i + 1] - offsets[i]) / 1000)
|
|
192
|
+
: 1 / fps; // last frame duration = 1 / avg fps
|
|
193
|
+
lines.push(`file 'frame_${String(i).padStart(6, '0')}.jpg'`);
|
|
194
|
+
lines.push(`duration ${durationSec.toFixed(6)}`);
|
|
195
|
+
}
|
|
196
|
+
// ffmpeg concat demuxer quirk: last `file` entry is needed without a
|
|
197
|
+
// `duration` directive to close the list correctly.
|
|
198
|
+
lines.push(`file 'frame_${String(offsets.length - 1).padStart(6, '0')}.jpg'`);
|
|
199
|
+
const concatPath = path.join(opts.framesDir, 'concat.txt');
|
|
200
|
+
await (await import('node:fs/promises')).writeFile(concatPath, lines.join('\n'));
|
|
201
|
+
await execFileAsync(ffmpeg, [
|
|
202
|
+
'-f', 'concat',
|
|
203
|
+
'-safe', '0',
|
|
204
|
+
'-i', concatPath,
|
|
205
|
+
'-fps_mode', 'vfr',
|
|
206
|
+
'-c:v', 'libx264',
|
|
207
|
+
'-preset', 'medium',
|
|
208
|
+
'-crf', '16',
|
|
209
|
+
'-pix_fmt', 'yuv420p',
|
|
210
|
+
'-movflags', '+faststart',
|
|
211
|
+
'-an',
|
|
212
|
+
'-y', opts.outputPath,
|
|
213
|
+
]);
|
|
214
|
+
return;
|
|
215
|
+
}
|
|
216
|
+
// CFR fallback (uniform framerate).
|
|
217
|
+
const pattern = path.join(opts.framesDir, 'frame_%06d.jpg');
|
|
218
|
+
await execFileAsync(ffmpeg, [
|
|
219
|
+
'-framerate', String(fps),
|
|
220
|
+
'-i', pattern,
|
|
221
|
+
'-c:v', 'libx264',
|
|
222
|
+
'-preset', 'medium',
|
|
223
|
+
'-crf', '16',
|
|
224
|
+
'-pix_fmt', 'yuv420p',
|
|
225
|
+
'-movflags', '+faststart',
|
|
226
|
+
'-an',
|
|
227
|
+
'-y', opts.outputPath,
|
|
228
|
+
]);
|
|
229
|
+
}
|
|
106
230
|
/**
|
|
107
231
|
* Trim a recording: skip dead frames at the start and cap duration.
|
|
108
232
|
*/
|
|
@@ -148,54 +272,47 @@ async function freezeLastFrame(inputPath, outputPath, durationSec) {
|
|
|
148
272
|
}
|
|
149
273
|
/**
|
|
150
274
|
* Full post-processing pipeline for a single clip recording:
|
|
151
|
-
*
|
|
275
|
+
* encode a normalized master MP4 → derive GIF + thumbnail → measure.
|
|
152
276
|
*/
|
|
153
277
|
export async function postProcessClipRecording(webmPath, outputDir, clipId, options = {}) {
|
|
154
278
|
const maxDuration = options.maxDurationSec ?? DEFAULT_MAX_DURATION_SEC;
|
|
155
|
-
// Step 1: Trim the recording — cut the setup phase (page load, overlay dismiss)
|
|
156
279
|
const trimStart = options.trimStartSec ?? DEFAULT_TRIM_START_SEC;
|
|
157
|
-
const trimmedPath = path.join(outputDir, `${clipId}_trimmed.webm`);
|
|
158
|
-
await trimRecording(webmPath, trimmedPath, trimStart, maxDuration);
|
|
159
|
-
// Step 1b: Optionally freeze the last frame for a pause before looping
|
|
160
280
|
const holdSec = Math.min(Math.max(options.holdLastFrameSec ?? 0, 0), 10);
|
|
161
|
-
|
|
162
|
-
if (
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
281
|
+
const mp4Path = path.join(outputDir, `${clipId}.mp4`);
|
|
282
|
+
if (options.skipMp4Encode) {
|
|
283
|
+
// Source is already an H.264 MP4 produced client-side by ClipCaptureLoop.
|
|
284
|
+
// Copy it under the canonical name so the downstream GIF/thumbnail steps
|
|
285
|
+
// get a stable path. Skip re-encode to preserve quality and avoid ~1-2s
|
|
286
|
+
// of ffmpeg CPU.
|
|
287
|
+
await copyFile(webmPath, mp4Path);
|
|
166
288
|
}
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
289
|
+
else {
|
|
290
|
+
await convertToMp4(webmPath, mp4Path, {
|
|
291
|
+
width: options.mp4Width,
|
|
292
|
+
height: options.mp4Height,
|
|
293
|
+
trimStartSec: trimStart,
|
|
294
|
+
maxDurationSec: maxDuration,
|
|
295
|
+
holdLastFrameSec: holdSec,
|
|
296
|
+
});
|
|
297
|
+
}
|
|
298
|
+
const durationMs = await getMediaDurationMs(mp4Path);
|
|
299
|
+
const mp4Stat = await stat(mp4Path);
|
|
300
|
+
const result = {
|
|
301
|
+
durationMs,
|
|
302
|
+
mp4Path,
|
|
303
|
+
fileSizeBytes: mp4Stat.size,
|
|
304
|
+
};
|
|
172
305
|
const gifPath = path.join(outputDir, `${clipId}.gif`);
|
|
173
|
-
await convertToGif(
|
|
306
|
+
await convertToGif(mp4Path, gifPath, {
|
|
174
307
|
fps: options.gifFps,
|
|
175
308
|
maxWidth: options.gifMaxWidth,
|
|
176
309
|
loop: options.loop,
|
|
177
310
|
});
|
|
178
311
|
result.gifPath = gifPath;
|
|
179
|
-
const gifStat = await stat(gifPath);
|
|
180
|
-
result.fileSizeBytes = gifStat.size;
|
|
181
|
-
const mp4Path = path.join(outputDir, `${clipId}.mp4`);
|
|
182
|
-
await convertToMp4(sourcePath, mp4Path);
|
|
183
|
-
result.mp4Path = mp4Path;
|
|
184
312
|
// Step 3: Extract thumbnail
|
|
185
313
|
const thumbnailPath = path.join(outputDir, `${clipId}_thumb.png`);
|
|
186
|
-
await extractThumbnail(
|
|
314
|
+
await extractThumbnail(mp4Path, thumbnailPath);
|
|
187
315
|
result.thumbnailPath = thumbnailPath;
|
|
188
|
-
// Clean up intermediate files
|
|
189
|
-
try {
|
|
190
|
-
await rm(trimmedPath, { force: true });
|
|
191
|
-
}
|
|
192
|
-
catch { /* ignore */ }
|
|
193
|
-
if (sourcePath !== trimmedPath) {
|
|
194
|
-
try {
|
|
195
|
-
await rm(sourcePath, { force: true });
|
|
196
|
-
}
|
|
197
|
-
catch { /* ignore */ }
|
|
198
|
-
}
|
|
199
316
|
return result;
|
|
200
317
|
}
|
|
201
318
|
//# sourceMappingURL=clip-postprocess.js.map
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { ClipCaptureMethod } from './execution-types.js';
|
|
2
|
+
export declare const CLIP_BACKEND_ENV_VAR = "AUTOKAP_CLIP_BACKEND";
|
|
3
|
+
export declare const CLIP_HEADLESS_POLICY_ENV_VAR = "AUTOKAP_CLIP_HEADLESS_POLICY";
|
|
4
|
+
export type ClipHeadlessPolicy = 'auto' | 'headless' | 'headed';
|
|
5
|
+
export interface ClipRuntimeSelection {
|
|
6
|
+
requestedMethod: ClipCaptureMethod;
|
|
7
|
+
captureMethod: ClipCaptureMethod;
|
|
8
|
+
browserHeaded: boolean;
|
|
9
|
+
headlessPolicy: ClipHeadlessPolicy;
|
|
10
|
+
fallbackReason?: string;
|
|
11
|
+
browserLaunchArgs?: string[];
|
|
12
|
+
}
|
|
13
|
+
export declare function resolveRequestedClipCaptureMethod(): ClipCaptureMethod;
|
|
14
|
+
export declare function resolveClipHeadlessPolicy(): ClipHeadlessPolicy;
|
|
15
|
+
export declare function resolveClipRuntimeSelection(options?: {
|
|
16
|
+
headed?: boolean;
|
|
17
|
+
platform?: NodeJS.Platform;
|
|
18
|
+
}): Promise<ClipRuntimeSelection>;
|