autokap 1.8.5 → 1.8.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/action-verifier.d.ts +6 -0
- package/dist/action-verifier.js +30 -17
- package/dist/browser.d.ts +59 -0
- package/dist/browser.js +259 -0
- package/dist/cli-contract.d.ts +5 -0
- package/dist/cli-runner.d.ts +0 -1
- package/dist/cli-runner.js +74 -59
- package/dist/clip-capture-loop.d.ts +28 -7
- package/dist/clip-capture-loop.js +102 -19
- package/dist/engine-version.d.ts +24 -0
- package/dist/engine-version.js +25 -0
- package/dist/execution-schema.d.ts +22 -0
- package/dist/execution-schema.js +59 -8
- package/dist/execution-types.d.ts +116 -0
- package/dist/opcode-runner.d.ts +8 -1
- package/dist/opcode-runner.js +120 -29
- package/dist/postcondition.d.ts +18 -3
- package/dist/postcondition.js +75 -27
- package/dist/program-hash.d.ts +11 -0
- package/dist/program-hash.js +28 -0
- package/dist/program-migrations.d.ts +31 -0
- package/dist/program-migrations.js +93 -0
- package/dist/program-signing.d.ts +11 -0
- package/dist/program-signing.js +1 -0
- package/dist/recovery-chain.js +8 -11
- package/dist/scenario-cookie.d.ts +36 -0
- package/dist/scenario-cookie.js +62 -0
- package/dist/security.d.ts +21 -0
- package/dist/security.js +46 -8
- package/dist/version.d.ts +1 -0
- package/dist/version.js +1 -0
- package/dist/video-narration-schema.d.ts +3 -0
- package/dist/video-narration-schema.js +3 -0
- package/dist/wait-contract.d.ts +104 -0
- package/dist/wait-contract.js +144 -0
- package/dist/web-playwright-local.d.ts +9 -1
- package/dist/web-playwright-local.js +0 -0
- package/package.json +2 -2
package/dist/cli-runner.js
CHANGED
|
@@ -29,14 +29,17 @@ import { applyDeviceFrame, seedDeviceConfigs } from './mockup.js';
|
|
|
29
29
|
import { transformBrowserUrl } from './transform-browser-url.js';
|
|
30
30
|
import { localizeStatusBar } from './status-bar-l10n.js';
|
|
31
31
|
import { logger } from './logger.js';
|
|
32
|
+
import { buildPreconditionCookies } from './scenario-cookie.js';
|
|
32
33
|
import { callLLM } from './llm-provider.js';
|
|
33
34
|
import { APP_VERSION } from './version.js';
|
|
35
|
+
import { ENGINE_VERSION, CURRENT_PROGRAM_SCHEMA_VERSION } from './engine-version.js';
|
|
36
|
+
import { readOriginSchemaVersion } from './program-migrations.js';
|
|
37
|
+
import { hashProgram } from './program-hash.js';
|
|
34
38
|
import { normalizeAllowedOrigins, normalizeHttpOrigin, verifySignedExecutionProgramEnvelope, } from './program-signing.js';
|
|
35
39
|
import { redactTelemetryText, redactUrl } from './telemetry-redaction.js';
|
|
36
40
|
import { LogCollector } from './log-collector.js';
|
|
37
41
|
const MAX_CLIP_CAPTURE_DEVICE_SCALE_FACTOR = 1;
|
|
38
42
|
const DEFAULT_VIDEO_DELIVERY_RESOLUTION = { width: 1920, height: 1080 };
|
|
39
|
-
const DEFAULT_VIDEO_CAPTURE_RESOLUTION = DEFAULT_VIDEO_DELIVERY_RESOLUTION;
|
|
40
43
|
const FETCH_PROGRAM_MAX_ATTEMPTS = 4;
|
|
41
44
|
const FETCH_PROGRAM_RETRY_DELAYS_MS = [1000, 3000, 5000];
|
|
42
45
|
const DEFAULT_SCREENSHOT_ARTIFACT_UPLOAD_CONCURRENCY = 4;
|
|
@@ -60,55 +63,25 @@ export function resolveRecordableBrowserSettings(program, variant) {
|
|
|
60
63
|
runtimeDeviceScaleFactor: 1,
|
|
61
64
|
};
|
|
62
65
|
}
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
const deliveryResolution = DEFAULT_VIDEO_DELIVERY_RESOLUTION;
|
|
68
|
-
const captureResolution = format.captureResolution ?? DEFAULT_VIDEO_CAPTURE_RESOLUTION;
|
|
69
|
-
// Variants are normalized too so any code path that reads `variant.viewport`
|
|
70
|
-
// or `variant.deviceScaleFactor` directly (not just resolveRecordableBrowserSettings)
|
|
71
|
-
// sees consistent 1920×1080 @1× values. Legacy presets carrying
|
|
72
|
-
// viewport=2560×1440 or DPR=1.3333 would otherwise leak through.
|
|
73
|
-
const targetViewport = { width: deliveryResolution.width, height: deliveryResolution.height };
|
|
74
|
-
const variantsAlreadyNormalized = program.variants.every((v) => v.viewport.width === targetViewport.width &&
|
|
75
|
-
v.viewport.height === targetViewport.height &&
|
|
76
|
-
(v.deviceScaleFactor === undefined || v.deviceScaleFactor === 1));
|
|
77
|
-
const formatAlreadyNormalized = captureResolution.width === deliveryResolution.width &&
|
|
78
|
-
captureResolution.height === deliveryResolution.height &&
|
|
79
|
-
format.deliveryResolution?.width === deliveryResolution.width &&
|
|
80
|
-
format.deliveryResolution.height === deliveryResolution.height;
|
|
81
|
-
const outputScaleAlreadyNormalized = program.outputScale === undefined || program.outputScale === 1;
|
|
82
|
-
if (variantsAlreadyNormalized && formatAlreadyNormalized && outputScaleAlreadyNormalized) {
|
|
83
|
-
return program;
|
|
84
|
-
}
|
|
85
|
-
return {
|
|
86
|
-
...program,
|
|
87
|
-
outputScale: 1,
|
|
88
|
-
variants: program.variants.map((v) => ({
|
|
89
|
-
...v,
|
|
90
|
-
viewport: { ...targetViewport },
|
|
91
|
-
deviceScaleFactor: 1,
|
|
92
|
-
})),
|
|
93
|
-
artifactPlan: {
|
|
94
|
-
...program.artifactPlan,
|
|
95
|
-
format: {
|
|
96
|
-
...format,
|
|
97
|
-
captureResolution: {
|
|
98
|
-
width: deliveryResolution.width,
|
|
99
|
-
height: deliveryResolution.height,
|
|
100
|
-
},
|
|
101
|
-
deliveryResolution,
|
|
102
|
-
},
|
|
103
|
-
},
|
|
104
|
-
};
|
|
105
|
-
}
|
|
66
|
+
// Legacy video resolution normalization (2560×1440 → 1920×1080 @1×) moved into
|
|
67
|
+
// migrate-on-read (program-migrations.ts, migrate_0→1). It now happens inside
|
|
68
|
+
// parseProgram BEFORE strict validation, so the runner only ever sees the
|
|
69
|
+
// canonical 1920×1080 video form — no runtime normalization pass is needed.
|
|
106
70
|
function normalizeNumericScale(value) {
|
|
107
71
|
if (typeof value !== 'number' || !Number.isFinite(value) || value <= 0) {
|
|
108
72
|
return 1;
|
|
109
73
|
}
|
|
110
74
|
return value;
|
|
111
75
|
}
|
|
76
|
+
function buildRunProvenance(program, schemaVersionOrigin) {
|
|
77
|
+
return {
|
|
78
|
+
engineVersion: ENGINE_VERSION,
|
|
79
|
+
programSchemaVersion: program.programSchemaVersion ?? CURRENT_PROGRAM_SCHEMA_VERSION,
|
|
80
|
+
programSchemaVersionOrigin: schemaVersionOrigin,
|
|
81
|
+
cliVersion: APP_VERSION,
|
|
82
|
+
programHash: hashProgram(program),
|
|
83
|
+
};
|
|
84
|
+
}
|
|
112
85
|
const HEALER_SYSTEM_PROMPT = 'You repair failed deterministic browser opcodes. Respond only with JSON.';
|
|
113
86
|
// ── Main entry point ────────────────────────────────────────────────
|
|
114
87
|
export async function runCapture(options) {
|
|
@@ -125,12 +98,16 @@ export async function runCapture(options) {
|
|
|
125
98
|
};
|
|
126
99
|
}
|
|
127
100
|
const config = await requireConfig();
|
|
128
|
-
// Step 1: Get the compiled program
|
|
101
|
+
// Step 1: Get the compiled program. Capture the stored FORM version BEFORE
|
|
102
|
+
// migrate-on-read (parseProgram) bumps it, so run provenance can record the
|
|
103
|
+
// origin schema version (0 = legacy / no version field).
|
|
129
104
|
let resolvedProgram;
|
|
105
|
+
let schemaVersionOrigin = 0;
|
|
130
106
|
if (options.program) {
|
|
131
107
|
let parsedProgram;
|
|
132
108
|
try {
|
|
133
|
-
|
|
109
|
+
schemaVersionOrigin = readOriginSchemaVersion(options.program);
|
|
110
|
+
parsedProgram = parseProgram(options.program);
|
|
134
111
|
}
|
|
135
112
|
catch (err) {
|
|
136
113
|
return { success: false, error: `program validation failed: ${err instanceof Error ? err.message : String(err)}` };
|
|
@@ -150,10 +127,14 @@ export async function runCapture(options) {
|
|
|
150
127
|
security: fetched.security,
|
|
151
128
|
};
|
|
152
129
|
try {
|
|
153
|
-
// Step 2: Validate the program fetched from the server.
|
|
130
|
+
// Step 2: Validate the program fetched from the server. The server reports
|
|
131
|
+
// the TRUE pre-migration origin via the envelope meta (the fetched program
|
|
132
|
+
// is already server-migrated); fall back to reading it locally for older
|
|
133
|
+
// servers that don't send it.
|
|
134
|
+
schemaVersionOrigin = fetched.schemaVersionOrigin ?? readOriginSchemaVersion(resolvedProgram.program);
|
|
154
135
|
resolvedProgram = {
|
|
155
136
|
...resolvedProgram,
|
|
156
|
-
program:
|
|
137
|
+
program: parseProgram(resolvedProgram.program),
|
|
157
138
|
};
|
|
158
139
|
}
|
|
159
140
|
catch (err) {
|
|
@@ -192,7 +173,7 @@ export async function runCapture(options) {
|
|
|
192
173
|
videoAudioAssets = prepareResult.audioAssets;
|
|
193
174
|
videoAudioAssetsByLocale = prepareResult.audioAssetsByLocale;
|
|
194
175
|
try {
|
|
195
|
-
program =
|
|
176
|
+
program = parseProgram(program);
|
|
196
177
|
}
|
|
197
178
|
catch (err) {
|
|
198
179
|
return { success: false, runId, error: `prepared video program validation failed: ${err instanceof Error ? err.message : String(err)}` };
|
|
@@ -321,7 +302,8 @@ export async function runCapture(options) {
|
|
|
321
302
|
variantId: 'run',
|
|
322
303
|
message: 'saving captures',
|
|
323
304
|
});
|
|
324
|
-
const
|
|
305
|
+
const provenance = buildRunProvenance(program, schemaVersionOrigin);
|
|
306
|
+
const uploadOutcome = await uploadResults(config, program, runResult, runId, provenance);
|
|
325
307
|
if (program.mediaMode === 'video' && runResult.success) {
|
|
326
308
|
await signalVideoComplete(config, program, runResult, uploadOutcome.runId, videoAudioAssets, videoAudioAssetsByLocale);
|
|
327
309
|
}
|
|
@@ -434,7 +416,12 @@ async function fetchProgram(config, presetId, environmentName) {
|
|
|
434
416
|
if (envelope.meta?.stale) {
|
|
435
417
|
logger.warn('[capture] Program needs regeneration — preset settings require missing or outdated opcodes. Regenerate before relying on this capture.');
|
|
436
418
|
}
|
|
437
|
-
return {
|
|
419
|
+
return {
|
|
420
|
+
success: true,
|
|
421
|
+
program: envelope.program,
|
|
422
|
+
security: envelope.security,
|
|
423
|
+
schemaVersionOrigin: envelope.meta?.programSchemaVersionOrigin,
|
|
424
|
+
};
|
|
438
425
|
}
|
|
439
426
|
return { success: false, error: 'failed to fetch program: retry attempts exhausted' };
|
|
440
427
|
}
|
|
@@ -694,7 +681,7 @@ async function postRunStart(config, runId, presetId, variantCount, env) {
|
|
|
694
681
|
logger.warn(`[capture] Run registration error: ${message}`);
|
|
695
682
|
}
|
|
696
683
|
}
|
|
697
|
-
async function uploadResults(config, program, result, runId
|
|
684
|
+
async function uploadResults(config, program, result, runId, provenance) {
|
|
698
685
|
const artifactJobs = result.variantResults.flatMap((variant) => {
|
|
699
686
|
const variantSpec = program.variants.find((entry) => entry.id === variant.variantId);
|
|
700
687
|
return variant.artifacts.map((artifact) => ({
|
|
@@ -709,7 +696,7 @@ async function uploadResults(config, program, result, runId = randomUUID()) {
|
|
|
709
696
|
logger.info(`[capture] Uploading ${totalArtifacts} capture artifacts with concurrency ${artifactUploadConcurrency}`);
|
|
710
697
|
}
|
|
711
698
|
await runWithConcurrency(artifactJobs, artifactUploadConcurrency, async (job, index) => {
|
|
712
|
-
await uploadArtifact(config, program, runId, totalArtifacts, index + 1, job);
|
|
699
|
+
await uploadArtifact(config, program, runId, totalArtifacts, index + 1, job, provenance);
|
|
713
700
|
});
|
|
714
701
|
// Strip binary buffers from artifacts before sending. The raw PNG/video
|
|
715
702
|
// buffers were already uploaded via /api/cli/artifacts above, and the
|
|
@@ -745,6 +732,12 @@ async function uploadResults(config, program, result, runId = randomUUID()) {
|
|
|
745
732
|
presetId: program.presetId,
|
|
746
733
|
programVersion: program.programVersion,
|
|
747
734
|
compileFingerprint: program.compileFingerprint,
|
|
735
|
+
// AutoKap Engine provenance (AUT-242) — persisted on capture_runs for debug.
|
|
736
|
+
engineVersion: provenance.engineVersion,
|
|
737
|
+
programSchemaVersion: provenance.programSchemaVersion,
|
|
738
|
+
programSchemaVersionOrigin: provenance.programSchemaVersionOrigin,
|
|
739
|
+
cliVersion: provenance.cliVersion,
|
|
740
|
+
programHash: provenance.programHash,
|
|
748
741
|
success: result.success,
|
|
749
742
|
mediaMode: program.mediaMode,
|
|
750
743
|
telemetry: result.telemetry,
|
|
@@ -864,13 +857,13 @@ function inferVariantLocale(variantId) {
|
|
|
864
857
|
function inferVariantTheme(variantId) {
|
|
865
858
|
return variantId.endsWith('-dark') ? 'dark' : 'light';
|
|
866
859
|
}
|
|
867
|
-
async function uploadArtifact(config, program, runId, totalArtifacts, uploadNumber, job) {
|
|
860
|
+
async function uploadArtifact(config, program, runId, totalArtifacts, uploadNumber, job, provenance) {
|
|
868
861
|
const { artifact, variant, variantSpec } = job;
|
|
869
862
|
const filename = buildArtifactFilename(program.presetId, variant.variantId, artifact);
|
|
870
863
|
const label = artifact.captureName ?? artifact.clipName ?? filename;
|
|
871
864
|
logger.info(`[capture] Exporting capture ${uploadNumber}/${totalArtifacts}: ${label}`);
|
|
872
865
|
if (process.env.AUTOKAP_USE_LEGACY_MULTIPART_UPLOADS === '1') {
|
|
873
|
-
await uploadArtifactMultipart(config, program, runId, job, filename);
|
|
866
|
+
await uploadArtifactMultipart(config, program, runId, job, filename, provenance);
|
|
874
867
|
return;
|
|
875
868
|
}
|
|
876
869
|
const prepared = await prepareDirectArtifactUpload({
|
|
@@ -879,6 +872,7 @@ async function uploadArtifact(config, program, runId, totalArtifacts, uploadNumb
|
|
|
879
872
|
artifact,
|
|
880
873
|
variant,
|
|
881
874
|
variantSpec,
|
|
875
|
+
provenance,
|
|
882
876
|
});
|
|
883
877
|
const uploadsUrl = `${config.apiBaseUrl}/api/cli/artifacts/uploads`;
|
|
884
878
|
const initResponse = await fetch(uploadsUrl, {
|
|
@@ -939,13 +933,19 @@ async function uploadArtifact(config, program, runId, totalArtifacts, uploadNumb
|
|
|
939
933
|
throw new Error(`artifact completion failed for ${variant.variantId}: ${await formatServerError(completeResponse, completeUrl)}`);
|
|
940
934
|
}
|
|
941
935
|
}
|
|
942
|
-
async function uploadArtifactMultipart(config, program, runId, job, filename) {
|
|
936
|
+
async function uploadArtifactMultipart(config, program, runId, job, filename, provenance) {
|
|
943
937
|
const { artifact, variant, variantSpec } = job;
|
|
944
938
|
const formData = new FormData();
|
|
945
939
|
formData.append('file', new Blob([new Uint8Array(artifact.buffer)], { type: artifact.mimeType }), filename);
|
|
946
940
|
formData.append('presetId', program.presetId);
|
|
947
941
|
formData.append('programVersion', String(program.programVersion));
|
|
948
942
|
formData.append('compileFingerprint', program.compileFingerprint);
|
|
943
|
+
// AutoKap Engine provenance (AUT-242).
|
|
944
|
+
formData.append('engineVersion', String(provenance.engineVersion));
|
|
945
|
+
formData.append('programSchemaVersion', String(provenance.programSchemaVersion));
|
|
946
|
+
formData.append('programSchemaVersionOrigin', String(provenance.programSchemaVersionOrigin));
|
|
947
|
+
formData.append('cliVersion', provenance.cliVersion);
|
|
948
|
+
formData.append('programHash', provenance.programHash);
|
|
949
949
|
formData.append('runId', runId);
|
|
950
950
|
formData.append('variantId', variant.variantId);
|
|
951
951
|
formData.append('targetId', variantSpec?.targetId ?? variant.variantId);
|
|
@@ -1015,7 +1015,7 @@ async function uploadArtifactMultipart(config, program, runId, job, filename) {
|
|
|
1015
1015
|
}
|
|
1016
1016
|
}
|
|
1017
1017
|
async function prepareDirectArtifactUpload(params) {
|
|
1018
|
-
const { program, runId, artifact, variant, variantSpec } = params;
|
|
1018
|
+
const { program, runId, artifact, variant, variantSpec, provenance } = params;
|
|
1019
1019
|
const requestedDeviceScaleFactor = variantSpec?.deviceScaleFactor ?? program.outputScale ?? 2;
|
|
1020
1020
|
const isFrameCapture = artifact.mediaMode === 'clip' || artifact.mediaMode === 'video';
|
|
1021
1021
|
const deviceScaleFactor = isFrameCapture && Number.isFinite(requestedDeviceScaleFactor)
|
|
@@ -1026,6 +1026,11 @@ async function prepareDirectArtifactUpload(params) {
|
|
|
1026
1026
|
presetId: program.presetId,
|
|
1027
1027
|
programVersion: program.programVersion,
|
|
1028
1028
|
compileFingerprint: program.compileFingerprint,
|
|
1029
|
+
engineVersion: provenance.engineVersion,
|
|
1030
|
+
programSchemaVersion: provenance.programSchemaVersion,
|
|
1031
|
+
programSchemaVersionOrigin: provenance.programSchemaVersionOrigin,
|
|
1032
|
+
cliVersion: provenance.cliVersion,
|
|
1033
|
+
programHash: provenance.programHash,
|
|
1029
1034
|
runId,
|
|
1030
1035
|
variantId: variant.variantId,
|
|
1031
1036
|
targetId: variantSpec?.targetId ?? variant.variantId,
|
|
@@ -1332,8 +1337,18 @@ async function applyPreconditions(browser, program) {
|
|
|
1332
1337
|
// {{email}}/{{password}}/{{loginUrl}} placeholders inside opcodes by the
|
|
1333
1338
|
// opcode runner, and the program performs the UI login itself.
|
|
1334
1339
|
// - If neither is set, the run is anonymous.
|
|
1335
|
-
|
|
1336
|
-
|
|
1340
|
+
// Build seed cookies (+ the signed AUT-239 scenario switch). `secure` is
|
|
1341
|
+
// derived from the target protocol so local http captures aren't silently
|
|
1342
|
+
// stripped of their cookies (see buildPreconditionCookies).
|
|
1343
|
+
const { cookies, warning } = buildPreconditionCookies(preconditions, program.baseUrl, process.env.AUTOKAP_SCENARIO_SECRET);
|
|
1344
|
+
if (warning)
|
|
1345
|
+
logger.warn(`[capture] ${warning}`);
|
|
1346
|
+
if (preconditions.scenario && process.env.AUTOKAP_SCENARIO_SECRET) {
|
|
1347
|
+
logger.info(`[capture] Injected scenario "${preconditions.scenario}" cookie — the target app must run with ` +
|
|
1348
|
+
`AUTOKAP_SCENARIOS=1 (and the same AUTOKAP_SCENARIO_SECRET) or it will capture real data`);
|
|
1349
|
+
}
|
|
1350
|
+
if (cookies.length > 0) {
|
|
1351
|
+
await browser.addCookies(cookies);
|
|
1337
1352
|
}
|
|
1338
1353
|
if (preconditions.sessionStorage && Object.keys(preconditions.sessionStorage).length > 0) {
|
|
1339
1354
|
await browser.prepareSessionStorage(preconditions.sessionStorage, { replace: false });
|
|
@@ -1,12 +1,18 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* ClipCaptureLoop —
|
|
2
|
+
* ClipCaptureLoop — frame source for local clip/video recording.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
* `Page.
|
|
4
|
+
* Primary path (Playwright 1.59+): the official `page.screencast` API streams
|
|
5
|
+
* JPEG frames through Chromium's screencast pipeline via `onFrame`. Frames
|
|
6
|
+
* arrive on a more regular cadence than the previous pull-based
|
|
7
|
+
* `Page.captureScreenshot` loop, and start/stop is precise — frames are only
|
|
8
|
+
* delivered between `screencast.start()` and `screencast.stop()`, so an
|
|
9
|
+
* off-camera setup reload before recording is never filmed.
|
|
8
10
|
*
|
|
9
|
-
*
|
|
11
|
+
* Fallback path: if `page.screencast` is missing or fails to start (older
|
|
12
|
+
* Playwright, headless quirk), the recorder falls back to the legacy CDP
|
|
13
|
+
* `Page.captureScreenshot` loop (`optimizeForSpeed` + `fromSurface`).
|
|
14
|
+
*
|
|
15
|
+
* Either way, frames are buffered in memory (raw JPEG Buffers) and flushed to
|
|
10
16
|
* disk in parallel at `stop()`. `assembleMp4FromFrames` reads the per-frame
|
|
11
17
|
* timestamps and encodes VFR via the concat demuxer so playback matches the
|
|
12
18
|
* real capture cadence — essential when the compositor is CPU-bound on heavy
|
|
@@ -52,12 +58,14 @@ export interface ClipCaptureLoopResult {
|
|
|
52
58
|
* loop keeps trying, producing bursts and gaps).
|
|
53
59
|
*/
|
|
54
60
|
frameOffsetsMs: number[];
|
|
55
|
-
/** CDP Page.captureScreenshot wall time in milliseconds. */
|
|
61
|
+
/** CDP Page.captureScreenshot wall time in milliseconds (empty on screencast path). */
|
|
56
62
|
captureTimingMs: {
|
|
57
63
|
p50: number;
|
|
58
64
|
p95: number;
|
|
59
65
|
max: number;
|
|
60
66
|
};
|
|
67
|
+
/** True when frames came from the official page.screencast API; false on the CDP fallback. */
|
|
68
|
+
usedScreencast: boolean;
|
|
61
69
|
}
|
|
62
70
|
export declare class ClipCaptureLoop {
|
|
63
71
|
private readonly page;
|
|
@@ -69,6 +77,8 @@ export declare class ClipCaptureLoop {
|
|
|
69
77
|
private cdp;
|
|
70
78
|
private running;
|
|
71
79
|
private loopPromise;
|
|
80
|
+
private screencastActive;
|
|
81
|
+
private usedScreencast;
|
|
72
82
|
private frames;
|
|
73
83
|
private frameTimestamps;
|
|
74
84
|
private frameCaptureDurationsMs;
|
|
@@ -78,5 +88,16 @@ export declare class ClipCaptureLoop {
|
|
|
78
88
|
constructor(opts: ClipCaptureLoopOptions);
|
|
79
89
|
start(): Promise<void>;
|
|
80
90
|
stop(): Promise<ClipCaptureLoopResult>;
|
|
91
|
+
/**
|
|
92
|
+
* Common frame sink for both the screencast and CDP paths: stamps wall-clock
|
|
93
|
+
* timing for VFR and buffers the raw JPEG in memory.
|
|
94
|
+
*/
|
|
95
|
+
private handleFrame;
|
|
96
|
+
/**
|
|
97
|
+
* Device-pixel dimensions of the current surface, so screencast frames keep
|
|
98
|
+
* the HiDPI fidelity the CDP `fromSurface` path produced. Returns undefined
|
|
99
|
+
* (let screencast pick its default) if the page can't be evaluated.
|
|
100
|
+
*/
|
|
101
|
+
private resolveCaptureSize;
|
|
81
102
|
private loop;
|
|
82
103
|
}
|
|
@@ -1,12 +1,18 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* ClipCaptureLoop —
|
|
2
|
+
* ClipCaptureLoop — frame source for local clip/video recording.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
* `Page.
|
|
4
|
+
* Primary path (Playwright 1.59+): the official `page.screencast` API streams
|
|
5
|
+
* JPEG frames through Chromium's screencast pipeline via `onFrame`. Frames
|
|
6
|
+
* arrive on a more regular cadence than the previous pull-based
|
|
7
|
+
* `Page.captureScreenshot` loop, and start/stop is precise — frames are only
|
|
8
|
+
* delivered between `screencast.start()` and `screencast.stop()`, so an
|
|
9
|
+
* off-camera setup reload before recording is never filmed.
|
|
8
10
|
*
|
|
9
|
-
*
|
|
11
|
+
* Fallback path: if `page.screencast` is missing or fails to start (older
|
|
12
|
+
* Playwright, headless quirk), the recorder falls back to the legacy CDP
|
|
13
|
+
* `Page.captureScreenshot` loop (`optimizeForSpeed` + `fromSurface`).
|
|
14
|
+
*
|
|
15
|
+
* Either way, frames are buffered in memory (raw JPEG Buffers) and flushed to
|
|
10
16
|
* disk in parallel at `stop()`. `assembleMp4FromFrames` reads the per-frame
|
|
11
17
|
* timestamps and encodes VFR via the concat demuxer so playback matches the
|
|
12
18
|
* real capture cadence — essential when the compositor is CPU-bound on heavy
|
|
@@ -14,6 +20,7 @@
|
|
|
14
20
|
*/
|
|
15
21
|
import fs from 'node:fs/promises';
|
|
16
22
|
import path from 'node:path';
|
|
23
|
+
import { logger } from './logger.js';
|
|
17
24
|
export class ClipCaptureLoop {
|
|
18
25
|
page;
|
|
19
26
|
framesDir;
|
|
@@ -24,6 +31,8 @@ export class ClipCaptureLoop {
|
|
|
24
31
|
cdp = null;
|
|
25
32
|
running = false;
|
|
26
33
|
loopPromise = null;
|
|
34
|
+
screencastActive = false;
|
|
35
|
+
usedScreencast = false;
|
|
27
36
|
frames = [];
|
|
28
37
|
frameTimestamps = [];
|
|
29
38
|
frameCaptureDurationsMs = [];
|
|
@@ -56,13 +65,53 @@ export class ClipCaptureLoop {
|
|
|
56
65
|
this.minRestMs = Math.max(0, Math.min(250, opts.minRestMs ?? platformMinRest));
|
|
57
66
|
}
|
|
58
67
|
async start() {
|
|
59
|
-
this.cdp = await this.page.context().newCDPSession(this.page);
|
|
60
68
|
this.startedAt = performance.now();
|
|
61
69
|
this.running = true;
|
|
70
|
+
// Primary: official page.screencast (Playwright 1.59+). Chromium pushes
|
|
71
|
+
// JPEG frames as the page paints — more regular than pull-based capture,
|
|
72
|
+
// and start/stop bounds the window precisely.
|
|
73
|
+
//
|
|
74
|
+
// CRITICAL fidelity rule: the screencast stream downsamples to CSS
|
|
75
|
+
// resolution UNLESS `size` is the device-pixel surface. Without it we'd
|
|
76
|
+
// ship the blurry DevTools-grade stream the CDP `fromSurface` path was
|
|
77
|
+
// deliberately chosen to avoid. So we only take the screencast path when we
|
|
78
|
+
// could resolve that HiDPI size; if we can't (or screencast errors), we
|
|
79
|
+
// fall through to the full-surface CDP loop instead of capturing soft.
|
|
80
|
+
try {
|
|
81
|
+
const size = await this.resolveCaptureSize();
|
|
82
|
+
if (size) {
|
|
83
|
+
await this.page.screencast.start({
|
|
84
|
+
quality: this.jpegQuality,
|
|
85
|
+
size,
|
|
86
|
+
onFrame: (frame) => {
|
|
87
|
+
this.handleFrame(frame.data);
|
|
88
|
+
},
|
|
89
|
+
});
|
|
90
|
+
this.screencastActive = true;
|
|
91
|
+
this.usedScreencast = true;
|
|
92
|
+
return;
|
|
93
|
+
}
|
|
94
|
+
logger.warn('[capture] could not resolve the HiDPI capture size; using the CDP capture loop to preserve fidelity');
|
|
95
|
+
}
|
|
96
|
+
catch (err) {
|
|
97
|
+
logger.warn(`[capture] page.screencast unavailable, falling back to CDP capture loop: ${err instanceof Error ? err.message : String(err)}`);
|
|
98
|
+
}
|
|
99
|
+
// Fallback: legacy CDP Page.captureScreenshot loop. Captures the full
|
|
100
|
+
// device surface via `fromSurface` — full HiDPI fidelity, no size negotiation.
|
|
101
|
+
this.cdp = await this.page.context().newCDPSession(this.page);
|
|
62
102
|
this.loopPromise = this.loop();
|
|
63
103
|
}
|
|
64
104
|
async stop() {
|
|
65
105
|
this.running = false;
|
|
106
|
+
if (this.screencastActive) {
|
|
107
|
+
try {
|
|
108
|
+
await this.page.screencast.stop();
|
|
109
|
+
}
|
|
110
|
+
catch {
|
|
111
|
+
// page/context may already be closed.
|
|
112
|
+
}
|
|
113
|
+
this.screencastActive = false;
|
|
114
|
+
}
|
|
66
115
|
if (this.loopPromise) {
|
|
67
116
|
await this.loopPromise;
|
|
68
117
|
this.loopPromise = null;
|
|
@@ -76,10 +125,10 @@ export class ClipCaptureLoop {
|
|
|
76
125
|
}
|
|
77
126
|
this.cdp = null;
|
|
78
127
|
}
|
|
79
|
-
// Flush all captured
|
|
80
|
-
//
|
|
81
|
-
//
|
|
82
|
-
const writes = this.frames.map((data, i) => fs.writeFile(path.join(this.framesDir, `frame_${String(i).padStart(6, '0')}.jpg`), data
|
|
128
|
+
// Flush all captured frames to disk in parallel now that the loop has
|
|
129
|
+
// stopped. Doing this during capture would bottleneck on disk I/O;
|
|
130
|
+
// deferring gets us the full capture throughput.
|
|
131
|
+
const writes = this.frames.map((data, i) => fs.writeFile(path.join(this.framesDir, `frame_${String(i).padStart(6, '0')}.jpg`), data));
|
|
83
132
|
await Promise.all(writes);
|
|
84
133
|
const frameCount = this.frames.length;
|
|
85
134
|
const span = this.lastFrameAt - this.firstFrameAt;
|
|
@@ -89,6 +138,7 @@ export class ClipCaptureLoop {
|
|
|
89
138
|
// Snapshot offsets (ms from first frame) for VFR encoding downstream.
|
|
90
139
|
const frameOffsetsMs = this.frameTimestamps.map(ts => ts - this.firstFrameAt);
|
|
91
140
|
const captureTimingMs = summarizeTiming(this.frameCaptureDurationsMs);
|
|
141
|
+
const usedScreencast = this.usedScreencast;
|
|
92
142
|
// Release memory — the caller owns framesDir from here on.
|
|
93
143
|
this.frames = [];
|
|
94
144
|
this.frameTimestamps = [];
|
|
@@ -103,8 +153,46 @@ export class ClipCaptureLoop {
|
|
|
103
153
|
trimStartMs,
|
|
104
154
|
frameOffsetsMs,
|
|
105
155
|
captureTimingMs,
|
|
156
|
+
usedScreencast,
|
|
106
157
|
};
|
|
107
158
|
}
|
|
159
|
+
/**
|
|
160
|
+
* Common frame sink for both the screencast and CDP paths: stamps wall-clock
|
|
161
|
+
* timing for VFR and buffers the raw JPEG in memory.
|
|
162
|
+
*/
|
|
163
|
+
handleFrame(data) {
|
|
164
|
+
if (!this.running)
|
|
165
|
+
return;
|
|
166
|
+
const ts = performance.now();
|
|
167
|
+
if (this.firstFrameAt === 0)
|
|
168
|
+
this.firstFrameAt = ts;
|
|
169
|
+
this.lastFrameAt = ts;
|
|
170
|
+
this.frames.push(data);
|
|
171
|
+
this.frameTimestamps.push(ts);
|
|
172
|
+
}
|
|
173
|
+
/**
|
|
174
|
+
* Device-pixel dimensions of the current surface, so screencast frames keep
|
|
175
|
+
* the HiDPI fidelity the CDP `fromSurface` path produced. Returns undefined
|
|
176
|
+
* (let screencast pick its default) if the page can't be evaluated.
|
|
177
|
+
*/
|
|
178
|
+
async resolveCaptureSize() {
|
|
179
|
+
// Bounded so a wedged page can never stall recording start — the CDP path
|
|
180
|
+
// had no such pre-flight evaluate, so cap it and fall back to the default
|
|
181
|
+
// surface size on timeout/error.
|
|
182
|
+
const evalDims = this.page
|
|
183
|
+
.evaluate(() => ({
|
|
184
|
+
w: Math.round(window.innerWidth * window.devicePixelRatio),
|
|
185
|
+
h: Math.round(window.innerHeight * window.devicePixelRatio),
|
|
186
|
+
}))
|
|
187
|
+
.catch(() => null);
|
|
188
|
+
const dims = await Promise.race([
|
|
189
|
+
evalDims,
|
|
190
|
+
new Promise((resolve) => setTimeout(() => resolve(null), 1000)),
|
|
191
|
+
]);
|
|
192
|
+
if (dims && dims.w > 0 && dims.h > 0)
|
|
193
|
+
return { width: dims.w, height: dims.h };
|
|
194
|
+
return undefined;
|
|
195
|
+
}
|
|
108
196
|
async loop() {
|
|
109
197
|
while (this.running) {
|
|
110
198
|
if (!this.cdp)
|
|
@@ -127,14 +215,9 @@ export class ClipCaptureLoop {
|
|
|
127
215
|
return;
|
|
128
216
|
continue;
|
|
129
217
|
}
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
this.lastFrameAt = ts;
|
|
134
|
-
// Push raw base64 — no decode, no I/O. Keeps the capture loop tight.
|
|
135
|
-
// Decode+write happens in stop().
|
|
136
|
-
this.frames.push(data);
|
|
137
|
-
this.frameTimestamps.push(ts);
|
|
218
|
+
// Decode happens here (off the tight loop's hot path is the await above);
|
|
219
|
+
// store a Buffer so both paths share the same disk-flush in stop().
|
|
220
|
+
this.handleFrame(Buffer.from(data, 'base64'));
|
|
138
221
|
const elapsed = performance.now() - frameStartedAt;
|
|
139
222
|
this.frameCaptureDurationsMs.push(elapsed);
|
|
140
223
|
const restMs = Math.max(this.minRestMs, this.targetFrameIntervalMs - elapsed);
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AutoKap Engine — version axes (pure constants, safe on any runtime).
|
|
3
|
+
*
|
|
4
|
+
* Kept free of Node-only imports (no `node:module`/`package.json` reads) so it
|
|
5
|
+
* can be pulled into the schema validation chain that the web app bundles. The
|
|
6
|
+
* npm package version lives separately in version.ts (`APP_VERSION`).
|
|
7
|
+
*
|
|
8
|
+
* Two axes, both decoupled from the npm package version and from each other:
|
|
9
|
+
*
|
|
10
|
+
* - `CURRENT_PROGRAM_SCHEMA_VERSION` (FORM): the shape of an `ExecutionProgram`.
|
|
11
|
+
* Drives migrate-on-read (program-migrations.ts). A monotonic integer bumped
|
|
12
|
+
* whenever the program FORM changes in a way that needs a `migrate_vN→vN+1`
|
|
13
|
+
* function. Programs with no `programSchemaVersion` are treated as v0 (oldest).
|
|
14
|
+
* - `ENGINE_VERSION` (SEMANTICS / PROVENANCE): the version of the runtime engine.
|
|
15
|
+
* A monotonic integer bumped by hand whenever the engine's runtime semantics
|
|
16
|
+
* meaningfully change (e.g. a new wait model ships) — NOT a per-opcode switch:
|
|
17
|
+
* the engine always applies the current semantics to every program. Stamped on
|
|
18
|
+
* each run purely for debug ("which engine produced this screenshot").
|
|
19
|
+
*
|
|
20
|
+
* Both are orthogonal to `ExecutionProgram.programVersion`, which is a CONTENT
|
|
21
|
+
* revision counter bumped only by the healer (see program-patcher.ts).
|
|
22
|
+
*/
|
|
23
|
+
export declare const CURRENT_PROGRAM_SCHEMA_VERSION = 1;
|
|
24
|
+
export declare const ENGINE_VERSION = 1;
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* AutoKap Engine — version axes (pure constants, safe on any runtime).
|
|
3
|
+
*
|
|
4
|
+
* Kept free of Node-only imports (no `node:module`/`package.json` reads) so it
|
|
5
|
+
* can be pulled into the schema validation chain that the web app bundles. The
|
|
6
|
+
* npm package version lives separately in version.ts (`APP_VERSION`).
|
|
7
|
+
*
|
|
8
|
+
* Two axes, both decoupled from the npm package version and from each other:
|
|
9
|
+
*
|
|
10
|
+
* - `CURRENT_PROGRAM_SCHEMA_VERSION` (FORM): the shape of an `ExecutionProgram`.
|
|
11
|
+
* Drives migrate-on-read (program-migrations.ts). A monotonic integer bumped
|
|
12
|
+
* whenever the program FORM changes in a way that needs a `migrate_vN→vN+1`
|
|
13
|
+
* function. Programs with no `programSchemaVersion` are treated as v0 (oldest).
|
|
14
|
+
* - `ENGINE_VERSION` (SEMANTICS / PROVENANCE): the version of the runtime engine.
|
|
15
|
+
* A monotonic integer bumped by hand whenever the engine's runtime semantics
|
|
16
|
+
* meaningfully change (e.g. a new wait model ships) — NOT a per-opcode switch:
|
|
17
|
+
* the engine always applies the current semantics to every program. Stamped on
|
|
18
|
+
* each run purely for debug ("which engine produced this screenshot").
|
|
19
|
+
*
|
|
20
|
+
* Both are orthogonal to `ExecutionProgram.programVersion`, which is a CONTENT
|
|
21
|
+
* revision counter bumped only by the healer (see program-patcher.ts).
|
|
22
|
+
*/
|
|
23
|
+
export const CURRENT_PROGRAM_SCHEMA_VERSION = 1;
|
|
24
|
+
export const ENGINE_VERSION = 1;
|
|
25
|
+
//# sourceMappingURL=engine-version.js.map
|
|
@@ -1092,6 +1092,7 @@ export declare const PreconditionSpecSchema: z.ZodObject<{
|
|
|
1092
1092
|
domain: z.ZodString;
|
|
1093
1093
|
path: z.ZodOptional<z.ZodString>;
|
|
1094
1094
|
}, z.core.$strict>>>;
|
|
1095
|
+
scenario: z.ZodOptional<z.ZodString>;
|
|
1095
1096
|
}, z.core.$strict>;
|
|
1096
1097
|
export declare const ArtifactSpecSchema: z.ZodObject<{
|
|
1097
1098
|
mediaMode: z.ZodEnum<{
|
|
@@ -1131,6 +1132,8 @@ export declare const ArtifactSpecSchema: z.ZodObject<{
|
|
|
1131
1132
|
export declare const ExecutionProgramSchema: z.ZodObject<{
|
|
1132
1133
|
presetId: z.ZodString;
|
|
1133
1134
|
programVersion: z.ZodNumber;
|
|
1135
|
+
programSchemaVersion: z.ZodOptional<z.ZodNumber>;
|
|
1136
|
+
engineVersion: z.ZodOptional<z.ZodNumber>;
|
|
1134
1137
|
mediaMode: z.ZodEnum<{
|
|
1135
1138
|
video: "video";
|
|
1136
1139
|
clip: "clip";
|
|
@@ -1241,6 +1244,7 @@ export declare const ExecutionProgramSchema: z.ZodObject<{
|
|
|
1241
1244
|
domain: z.ZodString;
|
|
1242
1245
|
path: z.ZodOptional<z.ZodString>;
|
|
1243
1246
|
}, z.core.$strict>>>;
|
|
1247
|
+
scenario: z.ZodOptional<z.ZodString>;
|
|
1244
1248
|
}, z.core.$strict>;
|
|
1245
1249
|
steps: z.ZodArray<z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
1246
1250
|
url: z.ZodString;
|
|
@@ -4202,6 +4206,7 @@ export declare function safeParseProgramResult(data: unknown): z.ZodSafeParseRes
|
|
|
4202
4206
|
domain: string;
|
|
4203
4207
|
path?: string | undefined;
|
|
4204
4208
|
}[] | undefined;
|
|
4209
|
+
scenario?: string | undefined;
|
|
4205
4210
|
};
|
|
4206
4211
|
steps: ({
|
|
4207
4212
|
urlPattern: string;
|
|
@@ -4911,6 +4916,8 @@ export declare function safeParseProgramResult(data: unknown): z.ZodSafeParseRes
|
|
|
4911
4916
|
};
|
|
4912
4917
|
compileFingerprint: string;
|
|
4913
4918
|
compiledAt: string;
|
|
4919
|
+
programSchemaVersion?: number | undefined;
|
|
4920
|
+
engineVersion?: number | undefined;
|
|
4914
4921
|
maxParallelCaptures?: number | undefined;
|
|
4915
4922
|
outputScale?: number | undefined;
|
|
4916
4923
|
variantFingerprint?: string | undefined;
|
|
@@ -4930,3 +4937,18 @@ export declare function safeParseProgramResult(data: unknown): z.ZodSafeParseRes
|
|
|
4930
4937
|
publicUrl?: string | undefined;
|
|
4931
4938
|
environmentHttpHeaders?: Record<string, string> | undefined;
|
|
4932
4939
|
}>;
|
|
4940
|
+
export interface ClipNavigationViolation {
|
|
4941
|
+
/** Index of the offending NAVIGATE opcode in `program.steps`. */
|
|
4942
|
+
stepIndex: number;
|
|
4943
|
+
message: string;
|
|
4944
|
+
}
|
|
4945
|
+
/**
|
|
4946
|
+
* Returns every NAVIGATE opcode located between a BEGIN_CLIP and its END_CLIP.
|
|
4947
|
+
* Pure and mode-agnostic — mirrors the runtime clip window exactly.
|
|
4948
|
+
*/
|
|
4949
|
+
export declare function findNavigateInClipViolations(program: Pick<ExecutionProgram, 'steps'>): ClipNavigationViolation[];
|
|
4950
|
+
/**
|
|
4951
|
+
* Hard-rejection variant for authoring boundaries (preset create/update).
|
|
4952
|
+
* Throws with an actionable message if any NAVIGATE sits inside a clip.
|
|
4953
|
+
*/
|
|
4954
|
+
export declare function assertNoNavigateInClip(program: Pick<ExecutionProgram, 'steps'>): void;
|