clipwise 0.3.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.ko.md +15 -1
- package/README.md +15 -1
- package/dist/cli/index.js +862 -137
- package/dist/compose/frame-worker.js +142 -13
- package/dist/index.d.ts +306 -8
- package/dist/index.js +877 -48
- package/package.json +1 -1
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
// src/compose/frame-worker.ts
|
|
2
2
|
import { parentPort } from "worker_threads";
|
|
3
|
+
import sharp8 from "sharp";
|
|
3
4
|
|
|
4
5
|
// src/compose/compose-frame.ts
|
|
5
6
|
import sharp7 from "sharp";
|
|
@@ -202,6 +203,11 @@ async function applyMobileFrame(frameBuffer, deviceType, darkMode, frameWidth, f
|
|
|
202
203
|
{ input: maskedScreen, left: bezel.sides, top: bezel.top }
|
|
203
204
|
]).png().toBuffer();
|
|
204
205
|
}
|
|
206
|
+
async function buildBrowserChromeBuffer(viewportWidth, darkMode, dpr = 1) {
|
|
207
|
+
const tbarH = TITLE_BAR_HEIGHT * dpr;
|
|
208
|
+
const chromeSvg = buildBrowserChromeSvg(viewportWidth, darkMode, dpr);
|
|
209
|
+
return sharp(Buffer.from(chromeSvg)).resize(viewportWidth, tbarH).png().toBuffer();
|
|
210
|
+
}
|
|
205
211
|
async function applyDeviceFrame(frameBuffer, config, frameWidth, frameHeight, dpr = 1) {
|
|
206
212
|
if (!config.enabled || config.type === "none") return frameBuffer;
|
|
207
213
|
switch (config.type) {
|
|
@@ -383,6 +389,32 @@ function buildBackgroundSvg(config, width, height) {
|
|
|
383
389
|
<rect width="${width}" height="${height}" fill="url(#bg)"/>
|
|
384
390
|
</svg>`;
|
|
385
391
|
}
|
|
392
|
+
async function buildBackdropBuffer(config, outputWidth, outputHeight, extraOverlays = []) {
|
|
393
|
+
const padding = config.padding;
|
|
394
|
+
const contentWidth = outputWidth - padding * 2;
|
|
395
|
+
const contentHeight = outputHeight - padding * 2;
|
|
396
|
+
const bgSvg = buildBackgroundSvg(config, outputWidth, outputHeight);
|
|
397
|
+
const composites = [];
|
|
398
|
+
if (config.shadow && contentWidth > 0 && contentHeight > 0) {
|
|
399
|
+
const shadowSvg = Buffer.from(
|
|
400
|
+
`<svg xmlns="http://www.w3.org/2000/svg" width="${outputWidth}" height="${outputHeight}">
|
|
401
|
+
<defs>
|
|
402
|
+
<filter id="shadow" x="-20%" y="-20%" width="140%" height="140%">
|
|
403
|
+
<feDropShadow dx="0" dy="4" stdDeviation="16" flood-color="rgba(0,0,0,0.3)"/>
|
|
404
|
+
</filter>
|
|
405
|
+
</defs>
|
|
406
|
+
<rect x="${padding}" y="${padding}" width="${contentWidth}" height="${contentHeight}"
|
|
407
|
+
rx="${config.borderRadius}" ry="${config.borderRadius}" fill="rgba(0,0,0,0.15)" filter="url(#shadow)"/>
|
|
408
|
+
</svg>`
|
|
409
|
+
);
|
|
410
|
+
composites.push({ input: shadowSvg, left: 0, top: 0 });
|
|
411
|
+
}
|
|
412
|
+
for (const overlay of extraOverlays) {
|
|
413
|
+
composites.push({ input: overlay, left: 0, top: 0 });
|
|
414
|
+
}
|
|
415
|
+
const { data, info } = await sharp4(Buffer.from(bgSvg)).resize(outputWidth, outputHeight).composite(composites).ensureAlpha().raw().toBuffer({ resolveWithObject: true });
|
|
416
|
+
return { data: Buffer.from(data), width: info.width, height: info.height };
|
|
417
|
+
}
|
|
386
418
|
async function applyBackground(frameBuffer, config, outputWidth, outputHeight) {
|
|
387
419
|
const padding = config.padding;
|
|
388
420
|
const contentWidth = outputWidth - padding * 2;
|
|
@@ -477,8 +509,8 @@ async function renderKeystrokeHud(frameBuffer, keystrokes, frameTimestamp, confi
|
|
|
477
509
|
|
|
478
510
|
// src/effects/watermark.ts
|
|
479
511
|
import sharp6 from "sharp";
|
|
480
|
-
|
|
481
|
-
if (!config.enabled || !config.text) return
|
|
512
|
+
function buildWatermarkSvg(config, frameWidth, frameHeight) {
|
|
513
|
+
if (!config.enabled || !config.text) return "";
|
|
482
514
|
const charWidth = config.fontSize * 0.62;
|
|
483
515
|
const textWidth = Math.ceil(config.text.length * charWidth);
|
|
484
516
|
const margin = 16;
|
|
@@ -504,16 +536,47 @@ async function renderWatermark(frameBuffer, config, frameWidth, frameHeight) {
|
|
|
504
536
|
break;
|
|
505
537
|
}
|
|
506
538
|
const escaped = config.text.replace(/&/g, "&").replace(/</g, "<").replace(/>/g, ">").replace(/"/g, """);
|
|
507
|
-
|
|
539
|
+
return `<svg xmlns="http://www.w3.org/2000/svg" width="${frameWidth}" height="${frameHeight}" shape-rendering="geometricPrecision" text-rendering="geometricPrecision">
|
|
508
540
|
<text x="${x}" y="${y}"
|
|
509
541
|
font-family="system-ui, -apple-system, sans-serif" font-size="${config.fontSize}"
|
|
510
542
|
font-weight="600" fill="${config.color}"
|
|
511
543
|
opacity="${config.opacity.toFixed(3)}">${escaped}</text>
|
|
512
544
|
</svg>`;
|
|
513
|
-
|
|
545
|
+
}
|
|
546
|
+
async function renderWatermark(frameBuffer, config, frameWidth, frameHeight) {
|
|
547
|
+
if (!config.enabled || !config.text) return frameBuffer;
|
|
548
|
+
const svg = buildWatermarkSvg(config, frameWidth, frameHeight);
|
|
549
|
+
return sharp6(frameBuffer).composite([{ input: Buffer.from(svg), left: 0, top: 0 }]).png().toBuffer();
|
|
514
550
|
}
|
|
515
551
|
|
|
516
552
|
// src/compose/compose-frame.ts
|
|
553
|
+
async function buildStaticLayers(effects, output, viewportWidth, dpr) {
|
|
554
|
+
const wmSvg = buildWatermarkSvg(effects.watermark, output.width, output.height);
|
|
555
|
+
const extraOverlays = wmSvg ? [Buffer.from(wmSvg)] : [];
|
|
556
|
+
const { data, width, height } = await buildBackdropBuffer(
|
|
557
|
+
effects.background,
|
|
558
|
+
output.width,
|
|
559
|
+
output.height,
|
|
560
|
+
extraOverlays
|
|
561
|
+
);
|
|
562
|
+
let browserChromePng = null;
|
|
563
|
+
let browserChromeHeight = 0;
|
|
564
|
+
if (effects.deviceFrame.enabled && effects.deviceFrame.type === "browser") {
|
|
565
|
+
browserChromeHeight = TITLE_BAR_HEIGHT * dpr;
|
|
566
|
+
browserChromePng = await buildBrowserChromeBuffer(
|
|
567
|
+
viewportWidth,
|
|
568
|
+
effects.deviceFrame.darkMode,
|
|
569
|
+
dpr
|
|
570
|
+
);
|
|
571
|
+
}
|
|
572
|
+
return {
|
|
573
|
+
backdropRaw: data,
|
|
574
|
+
backdropWidth: width,
|
|
575
|
+
backdropHeight: height,
|
|
576
|
+
browserChromePng,
|
|
577
|
+
browserChromeHeight
|
|
578
|
+
};
|
|
579
|
+
}
|
|
517
580
|
function getFrameOffset(config, dpr = 1) {
|
|
518
581
|
if (!config.enabled) return { left: 0, top: 0 };
|
|
519
582
|
switch (config.type) {
|
|
@@ -541,7 +604,18 @@ async function composeFrame(frame, effects, output, context) {
|
|
|
541
604
|
cursorTrail: context?.cursorTrail ?? []
|
|
542
605
|
};
|
|
543
606
|
if (effects.deviceFrame.enabled) {
|
|
544
|
-
|
|
607
|
+
const sl2 = ctx.staticLayers;
|
|
608
|
+
if (sl2?.browserChromePng && effects.deviceFrame.type === "browser") {
|
|
609
|
+
buffer = await sharp7(buffer).extend({
|
|
610
|
+
top: sl2.browserChromeHeight,
|
|
611
|
+
bottom: 0,
|
|
612
|
+
left: 0,
|
|
613
|
+
right: 0,
|
|
614
|
+
background: { r: 0, g: 0, b: 0, alpha: 0 }
|
|
615
|
+
}).composite([{ input: sl2.browserChromePng, left: 0, top: 0 }]).png().toBuffer();
|
|
616
|
+
} else {
|
|
617
|
+
buffer = await applyDeviceFrame(buffer, effects.deviceFrame, width, height, dpr);
|
|
618
|
+
}
|
|
545
619
|
const meta2 = await sharp7(buffer).metadata();
|
|
546
620
|
width = meta2.width ?? width;
|
|
547
621
|
height = meta2.height ?? height;
|
|
@@ -609,18 +683,57 @@ async function composeFrame(frame, effects, output, context) {
|
|
|
609
683
|
};
|
|
610
684
|
buffer = await applyZoom(buffer, focusPoint, scale, width, height);
|
|
611
685
|
}
|
|
612
|
-
|
|
613
|
-
if (
|
|
614
|
-
|
|
686
|
+
const sl = ctx.staticLayers;
|
|
687
|
+
if (sl) {
|
|
688
|
+
const padding = effects.background.padding;
|
|
689
|
+
const contentWidth = output.width - padding * 2;
|
|
690
|
+
const contentHeight = output.height - padding * 2;
|
|
691
|
+
if (contentWidth > 0 && contentHeight > 0) {
|
|
692
|
+
const radius = effects.background.borderRadius;
|
|
693
|
+
const roundedMask = Buffer.from(
|
|
694
|
+
`<svg xmlns="http://www.w3.org/2000/svg" width="${contentWidth}" height="${contentHeight}">
|
|
695
|
+
<rect width="${contentWidth}" height="${contentHeight}" rx="${radius}" ry="${radius}" fill="#ffffff"/>
|
|
696
|
+
</svg>`
|
|
697
|
+
);
|
|
698
|
+
const { data: maskedData, info: maskedInfo } = await sharp7(buffer).resize(contentWidth, contentHeight, { fit: "fill" }).composite([{ input: roundedMask, blend: "dest-in" }]).ensureAlpha().raw().toBuffer({ resolveWithObject: true });
|
|
699
|
+
const { data: composited, info: compInfo } = await sharp7(sl.backdropRaw, {
|
|
700
|
+
raw: { width: sl.backdropWidth, height: sl.backdropHeight, channels: 4 }
|
|
701
|
+
}).composite([{
|
|
702
|
+
input: Buffer.from(maskedData),
|
|
703
|
+
raw: { width: maskedInfo.width, height: maskedInfo.height, channels: 4 },
|
|
704
|
+
left: padding,
|
|
705
|
+
top: padding
|
|
706
|
+
}]).raw().toBuffer({ resolveWithObject: true });
|
|
707
|
+
return {
|
|
708
|
+
index: frame.index,
|
|
709
|
+
buffer: Buffer.from(composited),
|
|
710
|
+
timestamp: frame.timestamp,
|
|
711
|
+
rawInfo: { width: compInfo.width, height: compInfo.height, channels: 4 }
|
|
712
|
+
};
|
|
713
|
+
}
|
|
714
|
+
buffer = sl.backdropRaw;
|
|
715
|
+
} else {
|
|
716
|
+
buffer = await applyBackground(buffer, effects.background, output.width, output.height);
|
|
717
|
+
if (effects.watermark.enabled) {
|
|
718
|
+
buffer = await renderWatermark(buffer, effects.watermark, output.width, output.height);
|
|
719
|
+
}
|
|
615
720
|
}
|
|
616
|
-
|
|
721
|
+
const { data: finalData, info: finalInfo } = await sharp7(buffer).resize(output.width, output.height, {
|
|
617
722
|
fit: "fill",
|
|
618
723
|
kernel: sharp7.kernel.lanczos3
|
|
619
|
-
}).
|
|
620
|
-
return {
|
|
724
|
+
}).ensureAlpha().raw().toBuffer({ resolveWithObject: true });
|
|
725
|
+
return {
|
|
726
|
+
index: frame.index,
|
|
727
|
+
buffer: Buffer.from(finalData),
|
|
728
|
+
timestamp: frame.timestamp,
|
|
729
|
+
rawInfo: { width: finalInfo.width, height: finalInfo.height, channels: 4 }
|
|
730
|
+
};
|
|
621
731
|
}
|
|
622
732
|
|
|
623
733
|
// src/compose/frame-worker.ts
|
|
734
|
+
sharp8.concurrency(1);
|
|
735
|
+
var cachedStaticLayers = null;
|
|
736
|
+
var cachedEffectsKey = "";
|
|
624
737
|
parentPort.on("message", async (msg) => {
|
|
625
738
|
try {
|
|
626
739
|
const { taskId, frame, effects, output, context } = msg;
|
|
@@ -628,12 +741,28 @@ parentPort.on("message", async (msg) => {
|
|
|
628
741
|
...frame,
|
|
629
742
|
screenshot: Buffer.from(frame.screenshot)
|
|
630
743
|
};
|
|
631
|
-
const
|
|
744
|
+
const effectsKey = `${output.width}x${output.height}`;
|
|
745
|
+
if (!cachedStaticLayers || cachedEffectsKey !== effectsKey) {
|
|
746
|
+
const meta = await sharp8(frameWithBuffer.screenshot).metadata();
|
|
747
|
+
const dpr = Math.round((meta.width ?? frame.viewport.width) / frame.viewport.width) || 1;
|
|
748
|
+
cachedStaticLayers = await buildStaticLayers(
|
|
749
|
+
effects,
|
|
750
|
+
output,
|
|
751
|
+
frame.viewport.width,
|
|
752
|
+
dpr
|
|
753
|
+
);
|
|
754
|
+
cachedEffectsKey = effectsKey;
|
|
755
|
+
}
|
|
756
|
+
const result = await composeFrame(frameWithBuffer, effects, output, {
|
|
757
|
+
...context,
|
|
758
|
+
staticLayers: cachedStaticLayers
|
|
759
|
+
});
|
|
632
760
|
const reply = {
|
|
633
761
|
taskId,
|
|
634
762
|
index: result.index,
|
|
635
763
|
timestamp: result.timestamp,
|
|
636
|
-
buffer: result.buffer
|
|
764
|
+
buffer: result.buffer,
|
|
765
|
+
rawInfo: result.rawInfo
|
|
637
766
|
};
|
|
638
767
|
parentPort.postMessage(reply);
|
|
639
768
|
} catch (err) {
|
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
|
+
import { EventEmitter } from 'events';
|
|
2
3
|
|
|
3
4
|
declare const StepActionSchema: z.ZodDiscriminatedUnion<"action", [z.ZodObject<{
|
|
4
5
|
action: z.ZodLiteral<"navigate">;
|
|
@@ -1794,12 +1795,45 @@ interface ComposedFrame {
|
|
|
1794
1795
|
index: number;
|
|
1795
1796
|
buffer: Buffer;
|
|
1796
1797
|
timestamp: number;
|
|
1798
|
+
/**
|
|
1799
|
+
* Present when buffer contains raw RGBA pixels (not PNG).
|
|
1800
|
+
* Allows the encoder to skip the PNG-decode step and consume pixels directly.
|
|
1801
|
+
*/
|
|
1802
|
+
rawInfo?: {
|
|
1803
|
+
width: number;
|
|
1804
|
+
height: number;
|
|
1805
|
+
channels: 4;
|
|
1806
|
+
};
|
|
1807
|
+
}
|
|
1808
|
+
interface DedupStats {
|
|
1809
|
+
/** CDP로부터 수신한 원본 프레임 수 */
|
|
1810
|
+
received: number;
|
|
1811
|
+
/** 중복 제거 후 실제 저장된 고유 프레임 수 */
|
|
1812
|
+
stored: number;
|
|
1813
|
+
/** 중복으로 판단해 건너뛴 프레임 수 */
|
|
1814
|
+
skipped: number;
|
|
1797
1815
|
}
|
|
1798
1816
|
interface RecordingSession {
|
|
1799
1817
|
scenario: Scenario;
|
|
1800
1818
|
frames: CapturedFrame[];
|
|
1801
1819
|
startTime: number;
|
|
1802
1820
|
endTime?: number;
|
|
1821
|
+
/** 정적 프레임 중복 제거 통계 */
|
|
1822
|
+
dedupStats?: DedupStats;
|
|
1823
|
+
}
|
|
1824
|
+
/**
|
|
1825
|
+
* Handle returned by ClipwiseRecorder.recordToChannel().
|
|
1826
|
+
*
|
|
1827
|
+
* frameStream: async iterable of CapturedFrames as they are captured
|
|
1828
|
+
* (post-dedup, no FPS resampling — frames arrive at CDP capture rate).
|
|
1829
|
+
* Closes automatically when recording ends.
|
|
1830
|
+
*
|
|
1831
|
+
* done: resolves with the full RecordingSession (including FPS-resampled
|
|
1832
|
+
* frames) when recording has completely finished.
|
|
1833
|
+
*/
|
|
1834
|
+
interface RecordingHandle {
|
|
1835
|
+
frameStream: AsyncIterable<CapturedFrame>;
|
|
1836
|
+
done: Promise<RecordingSession>;
|
|
1803
1837
|
}
|
|
1804
1838
|
|
|
1805
1839
|
declare class ClipwiseRecorder {
|
|
@@ -1820,6 +1854,10 @@ declare class ClipwiseRecorder {
|
|
|
1820
1854
|
private cursorSpeed;
|
|
1821
1855
|
private firstContentTimestamp;
|
|
1822
1856
|
private pendingResponsePromises;
|
|
1857
|
+
private lastFrameSignature;
|
|
1858
|
+
private dedupStats;
|
|
1859
|
+
private frameChannel;
|
|
1860
|
+
private channelIndex;
|
|
1823
1861
|
/**
|
|
1824
1862
|
* Launch the browser and create a page with the scenario viewport.
|
|
1825
1863
|
*/
|
|
@@ -1837,6 +1875,26 @@ declare class ClipwiseRecorder {
|
|
|
1837
1875
|
* Execute the full scenario with continuous capture and return a RecordingSession.
|
|
1838
1876
|
*/
|
|
1839
1877
|
record(scenario: Scenario): Promise<RecordingSession>;
|
|
1878
|
+
/**
|
|
1879
|
+
* Start recording concurrently and return a RecordingHandle immediately.
|
|
1880
|
+
*
|
|
1881
|
+
* frameStream: yields CapturedFrames as each unique frame arrives from CDP
|
|
1882
|
+
* (post-dedup, sequential indices starting at 0, NO FPS resampling).
|
|
1883
|
+
* Closes when recording ends.
|
|
1884
|
+
*
|
|
1885
|
+
* done: resolves with the full RecordingSession (FPS-resampled) once
|
|
1886
|
+
* all steps have completed and the browser has been cleaned up.
|
|
1887
|
+
*
|
|
1888
|
+
* Use this with CanvasRenderer.composeStreamOnline() to overlap recording
|
|
1889
|
+
* time with composition time — total wall-clock ≈ max(recordingMs, composeMs).
|
|
1890
|
+
*/
|
|
1891
|
+
recordToChannel(scenario: Scenario): RecordingHandle;
|
|
1892
|
+
/**
|
|
1893
|
+
* Build a single CapturedFrame from a RawFrame in real-time.
|
|
1894
|
+
* Used by recordToChannel() to emit frames as they arrive.
|
|
1895
|
+
* Cursor/click data reflects the timeline up to this moment.
|
|
1896
|
+
*/
|
|
1897
|
+
private buildFrameOnline;
|
|
1840
1898
|
/**
|
|
1841
1899
|
* Wait for a given duration while forcing periodic repaints
|
|
1842
1900
|
* so CDP screencast keeps sending frames even on static pages.
|
|
@@ -1890,6 +1948,30 @@ declare class ClipwiseRecorder {
|
|
|
1890
1948
|
cleanup(): Promise<void>;
|
|
1891
1949
|
}
|
|
1892
1950
|
|
|
1951
|
+
/**
|
|
1952
|
+
* Pre-computed static layers that are identical for every frame in a session.
|
|
1953
|
+
*
|
|
1954
|
+
* backdropRaw: background gradient + shadow + watermark composited together at
|
|
1955
|
+
* output dimensions, stored as raw RGBA. Workers composite the per-frame
|
|
1956
|
+
* screenshot onto this buffer instead of re-generating the background SVG
|
|
1957
|
+
* and shadow SVG on every frame — eliminating ~3 PNG encode/decode cycles.
|
|
1958
|
+
*
|
|
1959
|
+
* browserChromePng: pre-rasterized browser chrome bar PNG. Applied via
|
|
1960
|
+
* Sharp's .extend() + .composite() in a single pipeline pass instead of the
|
|
1961
|
+
* current two-pass create-blank-canvas + composite pattern.
|
|
1962
|
+
*
|
|
1963
|
+
* Both are computed once per worker (first frame), then cached in memory for
|
|
1964
|
+
* all subsequent frames handled by that worker.
|
|
1965
|
+
*/
|
|
1966
|
+
interface StaticLayers {
|
|
1967
|
+
backdropRaw: Buffer;
|
|
1968
|
+
backdropWidth: number;
|
|
1969
|
+
backdropHeight: number;
|
|
1970
|
+
/** Pre-rasterized browser chrome bar PNG. Null when device frame is disabled or not "browser". */
|
|
1971
|
+
browserChromePng: Buffer | null;
|
|
1972
|
+
/** Pixel height of the chrome bar (0 when browserChromePng is null). */
|
|
1973
|
+
browserChromeHeight: number;
|
|
1974
|
+
}
|
|
1893
1975
|
interface FrameContext {
|
|
1894
1976
|
zoomScale: number;
|
|
1895
1977
|
clickProgress: number | null;
|
|
@@ -1897,6 +1979,8 @@ interface FrameContext {
|
|
|
1897
1979
|
x: number;
|
|
1898
1980
|
y: number;
|
|
1899
1981
|
}>;
|
|
1982
|
+
/** When present, skip redundant per-frame SVG generation for background/watermark/device-frame. */
|
|
1983
|
+
staticLayers?: StaticLayers;
|
|
1900
1984
|
}
|
|
1901
1985
|
|
|
1902
1986
|
declare class CanvasRenderer {
|
|
@@ -1932,6 +2016,77 @@ declare class CanvasRenderer {
|
|
|
1932
2016
|
* Apply speed ramping: slow down near actions, speed up during idle.
|
|
1933
2017
|
*/
|
|
1934
2018
|
private applySpeedRamp;
|
|
2019
|
+
/**
|
|
2020
|
+
* Returns true when no effect requires the full frame array upfront.
|
|
2021
|
+
*
|
|
2022
|
+
* When true, composeStreamOnline() can be used: frames are composited as they
|
|
2023
|
+
* arrive (no need to wait for all frames to be collected first).
|
|
2024
|
+
*
|
|
2025
|
+
* Currently the only blocking effect is speed ramp, which needs to scan all
|
|
2026
|
+
* frames to compute action-proximity indices. Zoom uses the window-based
|
|
2027
|
+
* calculateAdaptiveZoomInWindow() so it works with a rolling lookahead buffer.
|
|
2028
|
+
*/
|
|
2029
|
+
canStreamOnline(): boolean;
|
|
2030
|
+
/**
|
|
2031
|
+
* Online streaming compose — accepts an AsyncIterable of frames (e.g. from
|
|
2032
|
+
* ClipwiseRecorder.recordToChannel()) and begins compositing immediately,
|
|
2033
|
+
* without waiting for all frames to be collected.
|
|
2034
|
+
*
|
|
2035
|
+
* Each frame is dispatched to the worker pool as soon as its zoom lookahead
|
|
2036
|
+
* window is satisfied (i.e. when frame i + transitionFrames has arrived).
|
|
2037
|
+
* This creates a natural pipeline: recording produces frames while workers
|
|
2038
|
+
* consume them in parallel.
|
|
2039
|
+
*
|
|
2040
|
+
* Requires canStreamOnline() === true (speedRamp must be disabled).
|
|
2041
|
+
* Transitions (step boundaries with transition: fade) are applied inline
|
|
2042
|
+
* using the same applyTransitionsToStream() logic as composeStream().
|
|
2043
|
+
*/
|
|
2044
|
+
composeStreamOnline(source: AsyncIterable<CapturedFrame>): AsyncGenerator<ComposedFrame>;
|
|
2045
|
+
/**
|
|
2046
|
+
* Worker-pool online streaming: dispatches frame i to a worker as soon as
|
|
2047
|
+
* frame i + transitionFrames has arrived from the source.
|
|
2048
|
+
*
|
|
2049
|
+
* Uses a notify-on-progress pattern (same as streamWithWorkers) extended
|
|
2050
|
+
* with an intake coroutine that feeds the growing frames[] buffer.
|
|
2051
|
+
*/
|
|
2052
|
+
private streamOnlineWithWorkers;
|
|
2053
|
+
/**
|
|
2054
|
+
* Stream frame composition — yields ComposedFrames as workers finish,
|
|
2055
|
+
* in display order, so the encoder can start before all frames are composed.
|
|
2056
|
+
*
|
|
2057
|
+
* Same 4-pass structure as composeAll():
|
|
2058
|
+
* Pass 1 & 2 run upfront (need the full frame set).
|
|
2059
|
+
* Pass 3 streams via the worker pool (ordered yield).
|
|
2060
|
+
* Pass 4 transitions are buffered inline and applied at step boundaries.
|
|
2061
|
+
*/
|
|
2062
|
+
composeStream(frames: CapturedFrame[]): AsyncGenerator<ComposedFrame>;
|
|
2063
|
+
/**
|
|
2064
|
+
* Worker-pool streaming: dispatches frames to workers and yields results
|
|
2065
|
+
* in display order as soon as each frame is ready.
|
|
2066
|
+
*
|
|
2067
|
+
* Uses a notify-on-progress pattern to bridge event-driven workers
|
|
2068
|
+
* to an ordered AsyncGenerator without busy-polling.
|
|
2069
|
+
*/
|
|
2070
|
+
private streamWithWorkers;
|
|
2071
|
+
/**
|
|
2072
|
+
* Sequential streaming fallback for small frame counts where worker
|
|
2073
|
+
* thread overhead would exceed the parallelism benefit.
|
|
2074
|
+
*/
|
|
2075
|
+
private streamSequential;
|
|
2076
|
+
/**
|
|
2077
|
+
* Pre-compute [startIdx, endIdx] windows for every fade transition so that
|
|
2078
|
+
* applyTransitionsToStream can buffer only those frames.
|
|
2079
|
+
*/
|
|
2080
|
+
private getTransitionWindows;
|
|
2081
|
+
/**
|
|
2082
|
+
* Wrap a ComposedFrame stream with inline transition buffering.
|
|
2083
|
+
*
|
|
2084
|
+
* Non-transition frames are yielded immediately.
|
|
2085
|
+
* Frames inside a fade window are held until both endpoints are available,
|
|
2086
|
+
* then the crossfade is applied and all window frames are flushed in order.
|
|
2087
|
+
* A pending map maintains global display order across window boundaries.
|
|
2088
|
+
*/
|
|
2089
|
+
private applyTransitionsToStream;
|
|
1935
2090
|
/**
|
|
1936
2091
|
* Apply crossfade transitions at step boundaries where configured.
|
|
1937
2092
|
*/
|
|
@@ -1958,19 +2113,116 @@ declare function encodeGif(frames: ComposedFrame[], config: OutputConfig): Promi
|
|
|
1958
2113
|
* Requires ffmpeg to be installed and available in PATH.
|
|
1959
2114
|
*/
|
|
1960
2115
|
declare function encodeMp4(frames: ComposedFrame[], config: OutputConfig): Promise<Buffer>;
|
|
2116
|
+
/**
|
|
2117
|
+
* Streaming variant of encodeMp4 — accepts an AsyncIterable so frames can
|
|
2118
|
+
* be piped to FFmpeg as they arrive from the composition pipeline,
|
|
2119
|
+
* overlapping composition and encoding rather than waiting for all frames.
|
|
2120
|
+
*/
|
|
2121
|
+
declare function encodeMp4Stream(frames: AsyncIterable<ComposedFrame>, config: OutputConfig): Promise<Buffer>;
|
|
1961
2122
|
/**
|
|
1962
2123
|
* Save a sequence of composed frames as individual PNG files.
|
|
1963
2124
|
*/
|
|
1964
2125
|
declare function savePngSequence(frames: ComposedFrame[], config: OutputConfig): Promise<string[]>;
|
|
1965
2126
|
|
|
2127
|
+
/**
|
|
2128
|
+
* Emitted by StreamingSession after each frame is composed.
|
|
2129
|
+
*/
|
|
2130
|
+
interface PipelineProgress {
|
|
2131
|
+
/** Number of frames composed so far */
|
|
2132
|
+
composed: number;
|
|
2133
|
+
/** Total frames in the session */
|
|
2134
|
+
total: number;
|
|
2135
|
+
/** Completion percentage (0–100) */
|
|
2136
|
+
pct: number;
|
|
2137
|
+
}
|
|
2138
|
+
interface ConcurrentResult {
|
|
2139
|
+
/** Fully-encoded MP4 buffer. */
|
|
2140
|
+
buffer: Buffer;
|
|
2141
|
+
/** Full RecordingSession (FPS-resampled) returned when recording completed. */
|
|
2142
|
+
session: RecordingSession;
|
|
2143
|
+
}
|
|
2144
|
+
/**
|
|
2145
|
+
* ConcurrentSession overlaps recording and composition in the same process.
|
|
2146
|
+
*
|
|
2147
|
+
* While the recorder captures CDP screencast frames, the compositor begins
|
|
2148
|
+
* applying effects immediately — each frame is dispatched to the worker pool
|
|
2149
|
+
* as soon as its zoom lookahead window is satisfied.
|
|
2150
|
+
*
|
|
2151
|
+
* Total wall-clock time ≈ max(recordingMs, composeMs) instead of the sum.
|
|
2152
|
+
* Requires renderer.canStreamOnline() === true (speedRamp must be disabled).
|
|
2153
|
+
*
|
|
2154
|
+
* Emits 'progress' after each composed frame.
|
|
2155
|
+
* During recording the total is unknown (pct = -1); after recording ends and
|
|
2156
|
+
* composition finishes, a final event with pct = 100 is emitted.
|
|
2157
|
+
*
|
|
2158
|
+
* Usage:
|
|
2159
|
+
* const pipeline = new ConcurrentSession(recorder, scenario, renderer);
|
|
2160
|
+
* pipeline.on("progress", ({ composed, total, pct }) => {
|
|
2161
|
+
* spinner.text = total > 0
|
|
2162
|
+
* ? `Processing... ${composed}/${total} (${pct}%)`
|
|
2163
|
+
* : `Recording & composing... ${composed} frames`;
|
|
2164
|
+
* });
|
|
2165
|
+
* const { buffer, session } = await pipeline.run();
|
|
2166
|
+
*/
|
|
2167
|
+
declare class ConcurrentSession extends EventEmitter {
|
|
2168
|
+
private readonly recorder;
|
|
2169
|
+
private readonly scenario;
|
|
2170
|
+
private readonly renderer;
|
|
2171
|
+
constructor(recorder: ClipwiseRecorder, scenario: Scenario, renderer: CanvasRenderer);
|
|
2172
|
+
/**
|
|
2173
|
+
* Start recording and compositing concurrently.
|
|
2174
|
+
* Returns when both recording and encoding are complete.
|
|
2175
|
+
*/
|
|
2176
|
+
run(): Promise<ConcurrentResult>;
|
|
2177
|
+
}
|
|
2178
|
+
/**
|
|
2179
|
+
* StreamingSession bridges a recorded session to the compose+encode streaming
|
|
2180
|
+
* pipeline and emits fine-grained progress events.
|
|
2181
|
+
*
|
|
2182
|
+
* Emits:
|
|
2183
|
+
* 'progress' — after each frame is composed, with a PipelineProgress payload
|
|
2184
|
+
*
|
|
2185
|
+
* Usage:
|
|
2186
|
+
* const pipeline = new StreamingSession(session, renderer);
|
|
2187
|
+
* pipeline.on("progress", ({ composed, total, pct }) => {
|
|
2188
|
+
* spinner.text = `Composing & encoding... ${composed}/${total} (${pct}%)`;
|
|
2189
|
+
* });
|
|
2190
|
+
* const mp4Buffer = await pipeline.run();
|
|
2191
|
+
*/
|
|
2192
|
+
declare class StreamingSession extends EventEmitter {
|
|
2193
|
+
private readonly session;
|
|
2194
|
+
private readonly renderer;
|
|
2195
|
+
constructor(session: RecordingSession, renderer: CanvasRenderer);
|
|
2196
|
+
/** Total frames in the underlying recording session. */
|
|
2197
|
+
get totalFrames(): number;
|
|
2198
|
+
/**
|
|
2199
|
+
* Run the compose → encode pipeline.
|
|
2200
|
+
*
|
|
2201
|
+
* Composes frames via the worker pool (Phase 1-B streaming, ordered yield),
|
|
2202
|
+
* forwarding each to FFmpeg as it completes. Emits a 'progress' event after
|
|
2203
|
+
* every composed frame so callers can update a spinner or progress bar.
|
|
2204
|
+
*
|
|
2205
|
+
* @returns The fully-encoded MP4 as a Buffer.
|
|
2206
|
+
*/
|
|
2207
|
+
run(): Promise<Buffer>;
|
|
2208
|
+
}
|
|
2209
|
+
|
|
1966
2210
|
/**
|
|
1967
2211
|
* Calculate adaptive zoom scale based on proximity to click/action frames.
|
|
1968
2212
|
* Zooms in smoothly near important actions, stays at 1.0 during idle.
|
|
1969
2213
|
*
|
|
2214
|
+
* Scans only the ±transitionFrames influence window — frames outside that
|
|
2215
|
+
* range always produce 1.0 anyway, so scanning the full array is wasted work.
|
|
2216
|
+
* This reduces per-call cost from O(n) to O(transitionFrames).
|
|
2217
|
+
*
|
|
2218
|
+
* For bulk context calculation over many frames, prefer the lookup-based API:
|
|
2219
|
+
* buildZoomClickLookup() once → calculateAdaptiveZoomFromLookup() per frame
|
|
2220
|
+
* which achieves O(n + n·log k) instead of O(n·transitionFrames).
|
|
2221
|
+
*
|
|
1970
2222
|
* @param frames - Array of frames with optional clickPosition
|
|
1971
2223
|
* @param currentIndex - Index of the current frame
|
|
1972
2224
|
* @param maxScale - Peak zoom scale
|
|
1973
|
-
* @param transitionFrames -
|
|
2225
|
+
* @param transitionFrames - Half-width of the zoom influence window (frames)
|
|
1974
2226
|
* @returns Scale value for the current frame (1.0 = no zoom)
|
|
1975
2227
|
*/
|
|
1976
2228
|
declare function calculateAdaptiveZoom(frames: Array<{
|
|
@@ -1979,6 +2231,42 @@ declare function calculateAdaptiveZoom(frames: Array<{
|
|
|
1979
2231
|
y: number;
|
|
1980
2232
|
} | null;
|
|
1981
2233
|
}>, currentIndex: number, maxScale: number, transitionFrames: number): number;
|
|
2234
|
+
/**
|
|
2235
|
+
* Pre-extract the indices of all click frames in a single O(n) pass.
|
|
2236
|
+
* Pass the result to calculateAdaptiveZoomFromLookup() for O(log k) per-frame
|
|
2237
|
+
* zoom computation, instead of O(transitionFrames) per frame.
|
|
2238
|
+
*
|
|
2239
|
+
* @returns Sorted array of frame indices that carry a clickPosition.
|
|
2240
|
+
*/
|
|
2241
|
+
declare function buildZoomClickLookup(frames: ReadonlyArray<{
|
|
2242
|
+
clickPosition: unknown;
|
|
2243
|
+
}>): number[];
|
|
2244
|
+
/**
|
|
2245
|
+
* Calculate adaptive zoom scale using a pre-built click index lookup.
|
|
2246
|
+
* Binary-searches the lookup for the nearest click — O(log k) per call.
|
|
2247
|
+
*
|
|
2248
|
+
* Use buildZoomClickLookup() once before iterating, then call this per frame.
|
|
2249
|
+
*
|
|
2250
|
+
* @param clickLookup - Sorted array of click frame indices from buildZoomClickLookup()
|
|
2251
|
+
* @param currentIndex - Index of the frame being evaluated
|
|
2252
|
+
*/
|
|
2253
|
+
declare function calculateAdaptiveZoomFromLookup(clickLookup: readonly number[], currentIndex: number, maxScale: number, transitionFrames: number): number;
|
|
2254
|
+
/**
|
|
2255
|
+
* Calculate adaptive zoom scale using only a local window of frames.
|
|
2256
|
+
* Does NOT need the full frame array — only frames within
|
|
2257
|
+
* [currentIndex - transitionFrames, currentIndex + transitionFrames].
|
|
2258
|
+
*
|
|
2259
|
+
* This is the Phase 3-A compatible API: when composition runs concurrently
|
|
2260
|
+
* with recording, only the ±transitionFrames lookahead buffer needs to be
|
|
2261
|
+
* available before frame i can be composed.
|
|
2262
|
+
*
|
|
2263
|
+
* @param windowFrames - Slice of frames around currentIndex
|
|
2264
|
+
* @param windowStart - Absolute timeline index of windowFrames[0]
|
|
2265
|
+
* @param currentIndex - Absolute timeline index of the frame being composed
|
|
2266
|
+
*/
|
|
2267
|
+
declare function calculateAdaptiveZoomInWindow(windowFrames: ReadonlyArray<{
|
|
2268
|
+
clickPosition: unknown;
|
|
2269
|
+
}>, windowStart: number, currentIndex: number, maxScale: number, transitionFrames: number): number;
|
|
1982
2270
|
/**
|
|
1983
2271
|
* Calculate pan offset to keep a focus point centered when zoomed in.
|
|
1984
2272
|
* The offset defines the top-left corner of the visible crop region.
|
|
@@ -2021,17 +2309,27 @@ type KeystrokeConfig = EffectsConfig["keystroke"];
|
|
|
2021
2309
|
*/
|
|
2022
2310
|
declare function renderKeystrokeHud(frameBuffer: Buffer, keystrokes: KeystrokeEvent[], frameTimestamp: number, config: KeystrokeConfig, frameWidth: number, frameHeight: number, dpr?: number): Promise<Buffer>;
|
|
2023
2311
|
|
|
2312
|
+
type RawInfo = {
|
|
2313
|
+
width: number;
|
|
2314
|
+
height: number;
|
|
2315
|
+
channels: 4;
|
|
2316
|
+
};
|
|
2024
2317
|
/**
|
|
2025
2318
|
* Apply a crossfade transition between two frame buffers.
|
|
2026
2319
|
* Uses raw pixel weighted averaging for accurate blending.
|
|
2027
2320
|
*
|
|
2028
|
-
* @param fromBuffer
|
|
2029
|
-
* @param toBuffer
|
|
2030
|
-
* @param progress
|
|
2031
|
-
* @param width
|
|
2032
|
-
* @param height
|
|
2321
|
+
* @param fromBuffer - The outgoing frame (PNG or raw RGBA)
|
|
2322
|
+
* @param toBuffer - The incoming frame (PNG or raw RGBA)
|
|
2323
|
+
* @param progress - 0 = fully "from", 1 = fully "to"
|
|
2324
|
+
* @param width - Frame width
|
|
2325
|
+
* @param height - Frame height
|
|
2326
|
+
* @param fromRawInfo - Pass when fromBuffer is raw RGBA
|
|
2327
|
+
* @param toRawInfo - Pass when toBuffer is raw RGBA
|
|
2033
2328
|
*/
|
|
2034
|
-
declare function applyCrossfade(fromBuffer: Buffer, toBuffer: Buffer, progress: number, width: number, height: number): Promise<
|
|
2329
|
+
declare function applyCrossfade(fromBuffer: Buffer, toBuffer: Buffer, progress: number, width: number, height: number, fromRawInfo?: RawInfo, toRawInfo?: RawInfo): Promise<{
|
|
2330
|
+
buffer: Buffer;
|
|
2331
|
+
rawInfo: RawInfo;
|
|
2332
|
+
}>;
|
|
2035
2333
|
|
|
2036
2334
|
type WatermarkConfig = EffectsConfig["watermark"];
|
|
2037
2335
|
/**
|
|
@@ -2060,4 +2358,4 @@ interface ValidationResult {
|
|
|
2060
2358
|
*/
|
|
2061
2359
|
declare function validateScenario(scenario: Scenario): ValidationResult;
|
|
2062
2360
|
|
|
2063
|
-
export { CanvasRenderer, type CapturedFrame, ClipwiseRecorder, type ComposedFrame, type EffectsConfig, type FrameContext, type KeystrokeEvent, type OutputConfig, type RecordingSession, type Scenario, type Step, type StepAction, applyCrossfade, calculateAdaptiveZoom, calculatePanOffset, encodeGif, encodeMp4, lerpZoom, loadScenario, parseScenario, renderCursorHighlight, renderCursorTrail, renderKeystrokeHud, renderWatermark, savePngSequence, validateScenario };
|
|
2361
|
+
export { CanvasRenderer, type CapturedFrame, ClipwiseRecorder, type ComposedFrame, type ConcurrentResult, ConcurrentSession, type EffectsConfig, type FrameContext, type KeystrokeEvent, type OutputConfig, type PipelineProgress, type RecordingHandle, type RecordingSession, type Scenario, type Step, type StepAction, StreamingSession, applyCrossfade, buildZoomClickLookup, calculateAdaptiveZoom, calculateAdaptiveZoomFromLookup, calculateAdaptiveZoomInWindow, calculatePanOffset, encodeGif, encodeMp4, encodeMp4Stream, lerpZoom, loadScenario, parseScenario, renderCursorHighlight, renderCursorTrail, renderKeystrokeHud, renderWatermark, savePngSequence, validateScenario };
|