kaloko 0.4.10 → 0.4.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -0
- package/package.json +1 -1
- package/schemas/config.v1.json +28 -0
- package/schemas/scenario.v1.json +23 -1
- package/skills/kaloko/SKILL.md +10 -3
- package/skills/kaloko/reference/mcp.md +1 -0
- package/skills/kaloko/reference/recording.md +95 -0
- package/src/cli.mjs +91 -13
- package/src/lib/anim.mjs +125 -0
- package/src/lib/apps/recording-native.mjs +212 -0
- package/src/lib/build.mjs +60 -4
- package/src/lib/doctor.mjs +5 -1
- package/src/lib/import.mjs +1 -1
- package/src/lib/recording.mjs +373 -0
- package/src/lib/trace.mjs +109 -0
- package/src/lib/video-cli.mjs +209 -0
- package/src/lib/video.mjs +427 -0
- package/src/lib/webm.mjs +144 -0
- package/viewer/index.html +186 -7
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
/* global MediaRecorder, createImageBitmap, requestAnimationFrame */
|
|
2
|
+
// `kaloko video …` (batch D, #23): the walkthrough video of a run (step recordings stitched with a title card per
|
|
3
|
+
// step: its title and verdict), recordings and traces of a run, and what this machine can record with.
|
|
4
|
+
import { mkdtemp, rm, writeFile, stat, mkdir } from 'node:fs/promises'
|
|
5
|
+
import { tmpdir } from 'node:os'
|
|
6
|
+
import { join, resolve, relative, dirname } from 'node:path'
|
|
7
|
+
import { resolveRun, addCapture } from './run.mjs'
|
|
8
|
+
import { loadScenario } from './scenario.mjs'
|
|
9
|
+
import { buildManifest } from './build.mjs'
|
|
10
|
+
import { launchEphemeral } from './browser.mjs'
|
|
11
|
+
import { readJson, localized, UserError } from './util.mjs'
|
|
12
|
+
import { videoTools, encoderCaps, ffmpeg, preset, installHint, outputSize } from './video.mjs'
|
|
13
|
+
import { formatTrace } from './trace.mjs'
|
|
14
|
+
|
|
15
|
+
const FORMATS = ['mp4', 'webm', 'gif']
|
|
16
|
+
const VERDICT = {
|
|
17
|
+
en: { pass: 'Passed', fail: 'Failed', partial: 'Partly passed', skip: 'Not walked', step: (i, n) => `Step ${i} of ${n}` },
|
|
18
|
+
cs: { pass: 'Splněno', fail: 'Nesplněno', partial: 'Splněno zčásti', skip: 'Nekontrolováno', step: (i, n) => `Krok ${i} z ${n}` }
|
|
19
|
+
}
|
|
20
|
+
const fmtSize = (n) => n >= 1048576 ? `${(n / 1048576).toFixed(1)} MB` : `${Math.round(n / 1024)} kB`
|
|
21
|
+
const flowOrder = (nodes) => nodes.filter((n) => n.kind !== 'decision' && n.kind !== 'stack').sort((a, b) => a.col - b.col || a.lane - b.lane)
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* The segments of a walkthrough in flow order for one locale and viewport: a step's own recording, or its chapter of
|
|
25
|
+
* the flow recording. [{ node, title, status, file, from, to }] (from/to in seconds, null for the whole file).
|
|
26
|
+
*/
|
|
27
|
+
export function walkthroughPlan (manifest, { locale, viewport = null, steps = null } = {}) {
|
|
28
|
+
const loc = locale || manifest.scope?.locales?.[0]
|
|
29
|
+
const flow = manifest.recordings?.flow?.[loc]
|
|
30
|
+
const out = []
|
|
31
|
+
for (const n of flowOrder(manifest.nodes || [])) {
|
|
32
|
+
if (steps && !steps.includes(n.id)) continue
|
|
33
|
+
const vps = n.captures?.[loc] || {}
|
|
34
|
+
const vp = viewport && vps[viewport]?.rec ? viewport : Object.keys(vps).find((v) => vps[v]?.rec && (!viewport || v === viewport))
|
|
35
|
+
const rec = vp ? vps[vp].rec : (n.recordings || []).find((r) => r.locale === loc && (!viewport || r.viewport === viewport))
|
|
36
|
+
const status = n.by_locale?.[loc] || n.status || 'skip'
|
|
37
|
+
const title = localized(n.title, loc) || n.id
|
|
38
|
+
if (rec) { out.push({ node: n.id, title, status, file: rec.mp4 || rec.webm, from: null, to: null }); continue }
|
|
39
|
+
const ch = flow?.chapters?.find((c) => c.node === n.id)
|
|
40
|
+
if (ch) out.push({ node: n.id, title, status, file: flow.mp4 || flow.webm, from: ch.start / 1000, to: ch.capture / 1000 })
|
|
41
|
+
}
|
|
42
|
+
return out
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** ffmpeg arguments that stitch title cards and clips into one file. */
|
|
46
|
+
export function stitchArgs ({ parts, out, format, width, height, fps, card = 1.6, encoders = {} }) {
|
|
47
|
+
const inputs = []; const filters = []; let k = 0
|
|
48
|
+
for (const p of parts) {
|
|
49
|
+
if (p.card) inputs.push('-loop', '1', '-t', String(card), '-framerate', String(fps), '-i', p.card)
|
|
50
|
+
else inputs.push(...(p.from != null ? ['-ss', p.from.toFixed(3)] : []), ...(p.to != null ? ['-to', p.to.toFixed(3)] : []), '-i', p.file)
|
|
51
|
+
filters.push(`[${k}:v]scale=${width}:${height}:force_original_aspect_ratio=decrease,pad=${width}:${height}:(ow-iw)/2:(oh-ih)/2:color=0x151514,fps=${fps},format=yuv420p,setsar=1[v${k}]`)
|
|
52
|
+
k++
|
|
53
|
+
}
|
|
54
|
+
const joined = `${parts.map((_, i) => `[v${i}]`).join('')}concat=n=${parts.length}:v=1:a=0[out]`
|
|
55
|
+
if (format === 'gif') return [...inputs, '-filter_complex', `${filters.join(';')};${joined};[out]split[a][b];[a]palettegen=max_colors=128[p];[b][p]paletteuse=dither=bayer[g]`, '-map', '[g]', '-loop', '0', out]
|
|
56
|
+
const codec = format === 'webm'
|
|
57
|
+
? ['-c:v', encoders.vp9 || 'libvpx-vp9', '-deadline', 'good', '-cpu-used', '5', '-row-mt', '1', '-b:v', '0', '-crf', '36']
|
|
58
|
+
: encoders.h264 === 'libx264' || !encoders.h264 ? ['-c:v', 'libx264', '-preset', 'medium', '-crf', '26', '-movflags', '+faststart'] : ['-c:v', encoders.h264, '-b:v', '2M', '-movflags', '+faststart']
|
|
59
|
+
return [...inputs, '-filter_complex', `${filters.join(';')};${joined}`, '-map', '[out]', '-an', ...codec, '-g', String(fps * 2), out]
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/** Without ffmpeg: the browser plays the clips and title cards onto a canvas and MediaRecorder keeps it (real time). */
|
|
63
|
+
async function stitchInBrowser (page, { parts, width, height, fps, card }) {
|
|
64
|
+
const clips = []
|
|
65
|
+
for (const p of parts) {
|
|
66
|
+
if (p.card) clips.push({ card: p.cardData, ms: card * 1000 })
|
|
67
|
+
else clips.push({ video: (await import('node:fs/promises').then((fs) => fs.readFile(p.file))).toString('base64'), type: p.file.endsWith('.mp4') ? 'video/mp4' : 'video/webm', from: p.from, to: p.to })
|
|
68
|
+
}
|
|
69
|
+
return page.evaluate(async ({ clips, width, height, fps }) => {
|
|
70
|
+
const canvas = document.createElement('canvas'); canvas.width = width; canvas.height = height; document.body.append(canvas)
|
|
71
|
+
const g = canvas.getContext('2d')
|
|
72
|
+
const fromB64 = (s) => Uint8Array.from(atob(s), (c) => c.charCodeAt(0))
|
|
73
|
+
const stream = canvas.captureStream(fps)
|
|
74
|
+
const type = ['video/webm;codecs=vp9', 'video/webm;codecs=vp8', 'video/webm'].find((t) => MediaRecorder.isTypeSupported(t))
|
|
75
|
+
const rec = new MediaRecorder(stream, { mimeType: type, videoBitsPerSecond: 1_500_000 })
|
|
76
|
+
const chunks = []; rec.ondataavailable = (e) => { if (e.data.size) chunks.push(e.data) }
|
|
77
|
+
const fit = (w, h) => { const k = Math.min(width / w, height / h); return [(width - w * k) / 2, (height - h * k) / 2, w * k, h * k] }
|
|
78
|
+
rec.start(500)
|
|
79
|
+
for (const c of clips) {
|
|
80
|
+
g.fillStyle = '#151514'; g.fillRect(0, 0, width, height)
|
|
81
|
+
if (c.card) {
|
|
82
|
+
const bm = await createImageBitmap(new Blob([fromB64(c.card)], { type: 'image/png' }))
|
|
83
|
+
const end = performance.now() + c.ms
|
|
84
|
+
while (performance.now() < end) { g.drawImage(bm, 0, 0, width, height); await new Promise((resolve) => requestAnimationFrame(resolve)) }
|
|
85
|
+
continue
|
|
86
|
+
}
|
|
87
|
+
const v = document.createElement('video'); v.muted = true; v.playsInline = true
|
|
88
|
+
v.src = URL.createObjectURL(new Blob([fromB64(c.video)], { type: c.type }))
|
|
89
|
+
await new Promise((resolve, reject) => { v.onloadeddata = resolve; v.onerror = reject })
|
|
90
|
+
if (c.from != null) { v.currentTime = c.from; await new Promise((resolve) => { v.onseeked = resolve }) }
|
|
91
|
+
await v.play()
|
|
92
|
+
while (!v.ended && (c.to == null || v.currentTime < c.to)) { const [x, y, w, h] = fit(v.videoWidth, v.videoHeight); g.fillStyle = '#151514'; g.fillRect(0, 0, width, height); g.drawImage(v, x, y, w, h); await new Promise((resolve) => requestAnimationFrame(resolve)) }
|
|
93
|
+
v.pause(); URL.revokeObjectURL(v.src)
|
|
94
|
+
}
|
|
95
|
+
await new Promise((resolve) => { rec.onstop = resolve; rec.stop() })
|
|
96
|
+
const buf = new Uint8Array(await new Blob(chunks).arrayBuffer())
|
|
97
|
+
let s = ''; for (let i = 0; i < buf.length; i += 0x8000) s += String.fromCharCode.apply(null, buf.subarray(i, i + 0x8000))
|
|
98
|
+
return btoa(s)
|
|
99
|
+
}, { clips, width, height, fps })
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/** `kaloko video export`: the walkthrough of the current run (or --run) as MP4, WebM or GIF. */
|
|
103
|
+
export async function exportWalkthrough ({ config, root, args, say }) {
|
|
104
|
+
const run = await resolveRun({ root, output: config.output, run: args.run })
|
|
105
|
+
const scenario = await loadScenario(resolve(root, run.meta.scenario_file))
|
|
106
|
+
const manifest = await buildManifest({ run, scenario, config, root, catalog: [] })
|
|
107
|
+
const format = FORMATS.includes(args.format) ? args.format : 'mp4'
|
|
108
|
+
const lang = args.lang === 'cs' ? 'cs' : 'en'
|
|
109
|
+
const locale = typeof args.locale === 'string' ? args.locale : manifest.scope.locales[0]
|
|
110
|
+
const steps = typeof args.steps === 'string' ? args.steps.split(',').map((s) => s.trim()).filter(Boolean) : null
|
|
111
|
+
const plan = walkthroughPlan(manifest, { locale, viewport: typeof args.viewport === 'string' ? args.viewport : null, steps })
|
|
112
|
+
if (!plan.length) throw new UserError(`No recordings in run ${run.meta.id} for ${locale}. Walk with recordings first: kaloko walk --record (or --record flow), then export.`)
|
|
113
|
+
const tools = await videoTools()
|
|
114
|
+
if (!tools.ffmpeg && format !== 'webm') throw new UserError(`A walkthrough as ${format.toUpperCase()} needs ffmpeg (${installHint('ffmpeg')}). Without it, export WebM: kaloko video export --format webm (it plays the clips once in real time).`)
|
|
115
|
+
const P = preset(args.quality || 'medium')
|
|
116
|
+
const fps = format === 'gif' ? 10 : P.fps
|
|
117
|
+
const [width, height] = format === 'gif' ? outputSize(1280, 800, 800) : outputSize(1280, 800, Math.min(P.width, 1280))
|
|
118
|
+
const card = Math.max(0.5, Math.min(6, Number(args['card-seconds']) || 1.6))
|
|
119
|
+
const browser = await launchEphemeral(config)
|
|
120
|
+
const tmp = await mkdtemp(join(tmpdir(), 'kaloko-walkthrough-'))
|
|
121
|
+
const defaultOut = join(run.dir, 'attachments', `walkthrough.rec.${format}`)
|
|
122
|
+
const out = typeof args.out === 'string' ? resolve(args.out) : defaultOut
|
|
123
|
+
try {
|
|
124
|
+
const page = await browser.newPage()
|
|
125
|
+
await encoderCaps(page)
|
|
126
|
+
const V = VERDICT[lang]
|
|
127
|
+
const parts = []
|
|
128
|
+
const cardPng = async (lines, verdict) => {
|
|
129
|
+
const r = await page.evaluate((a) => window.__kenc.card(a), { width, height, lines, verdict })
|
|
130
|
+
const f = join(tmp, `card${parts.length}.png`)
|
|
131
|
+
await writeFile(f, Buffer.from(r.data, 'base64'))
|
|
132
|
+
return { card: f, cardData: r.data }
|
|
133
|
+
}
|
|
134
|
+
const s = manifest.summary || {}
|
|
135
|
+
parts.push(await cardPng([`${manifest.environment?.name || ''} · ${locale} · ${new Date(manifest.started_at).toISOString().slice(0, 10)}`, localized(manifest.scenario.title, locale) || manifest.scenario.id, `${plan.length} ${lang === 'cs' ? 'kroků se záznamem' : 'recorded steps'}${s.pass != null ? ` · ${s.pass} ${V.pass.toLowerCase()} · ${s.fail || 0} ${V.fail.toLowerCase()}` : ''}`], null))
|
|
136
|
+
plan.forEach((p, i) => { p.index = i + 1 })
|
|
137
|
+
for (const p of plan) {
|
|
138
|
+
parts.push(await cardPng([V.step(p.index, plan.length), p.title, p.node], { status: p.status, text: V[p.status] || p.status }))
|
|
139
|
+
parts.push({ file: join(run.dir, p.file), from: p.from, to: p.to })
|
|
140
|
+
}
|
|
141
|
+
await mkdir(dirname(out), { recursive: true })
|
|
142
|
+
if (tools.ffmpeg) {
|
|
143
|
+
say(`Stitching ${plan.length} recording(s) with title cards (${format.toUpperCase()}, ${width}×${height})…`)
|
|
144
|
+
await ffmpeg(tools, stitchArgs({ parts, out, format, width, height, fps, card, encoders: tools.ffmpeg.encoders }))
|
|
145
|
+
} else {
|
|
146
|
+
say(`No ffmpeg: playing ${plan.length} recording(s) through the browser in real time to record a WebM…`)
|
|
147
|
+
await writeFile(out, Buffer.from(await stitchInBrowser(page, { parts, width, height, fps, card }), 'base64'))
|
|
148
|
+
}
|
|
149
|
+
const bytes = (await stat(out)).size
|
|
150
|
+
const duration = Math.round((plan.reduce((n, p) => n + ((p.to ?? 0) - (p.from ?? 0)), 0) + card * (plan.length + 1)) * 1000)
|
|
151
|
+
const inside = !relative(run.dir, out).startsWith('..')
|
|
152
|
+
if (inside) await addCapture(run.dir, { type: 'walkthrough', file: relative(run.dir, out), format, bytes, locale, viewport: args.viewport || null, steps: plan.length, ...(plan.every((p) => p.to != null) ? { duration_ms: duration } : {}), at: new Date().toISOString() })
|
|
153
|
+
say(`Walkthrough: ${relative(process.cwd(), out)} (${fmtSize(bytes)}, ${plan.length} steps)`)
|
|
154
|
+
if (inside) say('kaloko share uploads it with the run (Business plan and up); people with the link play or download it on the canvas.')
|
|
155
|
+
return { file: out, bytes, steps: plan.length }
|
|
156
|
+
} finally { await browser.close().catch(() => {}); await rm(tmp, { recursive: true, force: true }) }
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
export async function videoCmd ({ config, root, args, say }) {
|
|
160
|
+
const sub = args._[0] || 'help'
|
|
161
|
+
if (sub === 'export') return exportWalkthrough({ config, root, args, say })
|
|
162
|
+
if (sub === 'tools') {
|
|
163
|
+
const t = await videoTools({ refresh: true })
|
|
164
|
+
const ff = t.ffmpeg
|
|
165
|
+
say([
|
|
166
|
+
`ffmpeg ${ff ? `${ff.version} · H.264 ${ff.encoders.h264 || '—'} · VP9 ${ff.encoders.vp9 ? 'yes' : '—'} · AV1 ${ff.encoders.av1 || '—'}` : `not found (${installHint('ffmpeg', t.platform)}); WebM still works, MP4 and the walkthrough export need it`}`,
|
|
167
|
+
`adb ${t.adb ? 'found (Android screenrecord)' : `not found (${installHint('adb', t.platform)})`}`,
|
|
168
|
+
...(t.platform === 'darwin' ? [`simctl ${t.simctl ? 'found (iOS Simulator recordVideo)' : `not found (${installHint('simctl', 'darwin')})`}`, 'screencapture macOS desktop recordings (needs Screen Recording permission: kaloko doctor checks it)'] : [])
|
|
169
|
+
].join('\n'))
|
|
170
|
+
return t
|
|
171
|
+
}
|
|
172
|
+
const run = await resolveRun({ root, output: config.output, run: args.run })
|
|
173
|
+
const scenario = await loadScenario(resolve(root, run.meta.scenario_file))
|
|
174
|
+
const manifest = await buildManifest({ run, scenario, config, root, catalog: [] })
|
|
175
|
+
if (sub === 'list') {
|
|
176
|
+
const rows = []
|
|
177
|
+
for (const n of manifest.nodes) {
|
|
178
|
+
for (const [l, vps] of Object.entries(n.captures || {})) for (const [vp, c] of Object.entries(vps || {})) if (c?.rec || c?.trace_summary) rows.push({ step: n.id, locale: l, viewport: vp, recording: c.rec || null, trace: c.trace_summary || null })
|
|
179
|
+
for (const r of n.recordings || []) rows.push({ step: n.id, locale: r.locale, viewport: r.viewport, recording: r, trace: null })
|
|
180
|
+
}
|
|
181
|
+
const flows = manifest.recordings?.flow || {}
|
|
182
|
+
if (args.json) { say(JSON.stringify({ run: run.meta.id, steps: rows, flow: flows, walkthrough: manifest.walkthrough || null }, null, 2)); return }
|
|
183
|
+
for (const r of rows) say(`${r.step.padEnd(24)} ${r.locale}/${r.viewport} ${r.recording ? `${Math.round(r.recording.duration_ms / 100) / 10} s ${[r.recording.webm && 'WebM', r.recording.mp4 && 'MP4'].filter(Boolean).join('+')} ${fmtSize(r.recording.bytes || 0)}${r.recording.failed ? ' (failed attempt)' : ''}` : 'no recording'}${r.trace ? ` · trace ${r.trace.actions} actions, ${r.trace.requests} requests${r.trace.failed_requests ? ` (${r.trace.failed_requests} failed)` : ''}, ${r.trace.console_errors} console errors` : ''}`)
|
|
184
|
+
for (const [l, f] of Object.entries(flows)) say(`flow ${l}: ${Math.round(f.duration_ms / 1000)} s, ${f.chapters?.length || 0} chapters, ${fmtSize(f.bytes || 0)}`)
|
|
185
|
+
if (manifest.walkthrough) say(`walkthrough: ${manifest.walkthrough.file} (${fmtSize(manifest.walkthrough.bytes || 0)})`)
|
|
186
|
+
if (!rows.length && !Object.keys(flows).length) say('Nothing recorded in this run. kaloko walk --record records each step; traces are on unless capture.trace is false.')
|
|
187
|
+
return
|
|
188
|
+
}
|
|
189
|
+
if (sub === 'trace') {
|
|
190
|
+
const step = typeof args.step === 'string' ? args.step : null
|
|
191
|
+
const n = manifest.nodes.find((x) => x.id === step)
|
|
192
|
+
if (!n) throw new UserError(`Usage: kaloko video trace --step <id> [--locale l] [--viewport v] [--json]. Steps: ${manifest.nodes.map((x) => x.id).join(', ')}`)
|
|
193
|
+
const loc = typeof args.locale === 'string' ? args.locale : Object.keys(n.captures || {})[0]
|
|
194
|
+
const vps = n.captures?.[loc] || {}
|
|
195
|
+
const vp = typeof args.viewport === 'string' ? args.viewport : Object.keys(vps).find((v) => vps[v]?.trace)
|
|
196
|
+
const file = vps[vp]?.trace
|
|
197
|
+
if (!file) throw new UserError(`No trace for ${step} ${loc}${vp ? `/${vp}` : ''}. Walk the step again (traces are on unless capture.trace is false).`)
|
|
198
|
+
const trace = await readJson(join(run.dir, file), null)
|
|
199
|
+
say(args.json ? JSON.stringify(trace, null, 2) : `${step} ${loc}/${vp}: ${trace.duration_ms} ms, ${trace.summary?.requests ?? 0} requests, ${trace.summary?.console_errors ?? 0} console errors\n${formatTrace(trace)}`)
|
|
200
|
+
return
|
|
201
|
+
}
|
|
202
|
+
say(`Usage:
|
|
203
|
+
kaloko video export [--format mp4|webm|gif] [--locale l] [--viewport v] [--steps a,b] [--lang en|cs] [--out file]
|
|
204
|
+
The run's recordings stitched in flow order with a title card per step (title, verdict)
|
|
205
|
+
kaloko video list [--json] Recordings and traces of the run, with sizes
|
|
206
|
+
kaloko video trace --step <id> [--locale l] [--viewport v] [--json]
|
|
207
|
+
Actions, requests and console messages of the step, in time order
|
|
208
|
+
kaloko video tools ffmpeg, adb, simctl: what this machine can record and encode with`)
|
|
209
|
+
}
|
|
@@ -0,0 +1,427 @@
|
|
|
1
|
+
/* global VideoEncoder, VideoFrame, OffscreenCanvas, createImageBitmap, location */
|
|
2
|
+
// Video tooling for recordings (batch D): quality presets, which formats this machine can make, the encoder that runs
|
|
3
|
+
// in Chrome (WebCodecs, no native dependency) and the optional ffmpeg steps (MP4 H.264 for Safari, AV1, native app
|
|
4
|
+
// recordings, walkthrough export). Without ffmpeg a web recording still becomes a seekable WebM (VP9 or VP8), a
|
|
5
|
+
// poster and an animated preview; MP4 and the stitched walkthrough need ffmpeg.
|
|
6
|
+
import { execFile, spawn } from 'node:child_process'
|
|
7
|
+
import { readFile, writeFile, mkdtemp, rm, stat, readdir } from 'node:fs/promises'
|
|
8
|
+
import { tmpdir } from 'node:os'
|
|
9
|
+
import { join } from 'node:path'
|
|
10
|
+
import { muxWebm } from './webm.mjs'
|
|
11
|
+
import { animatedWebp, encodeGif } from './anim.mjs'
|
|
12
|
+
|
|
13
|
+
/** Quality presets: frames per second, longest side, JPEG quality of screencast frames, bitrates and CRFs. */
|
|
14
|
+
export const PRESETS = {
|
|
15
|
+
low: { fps: 8, width: 960, jpeg: 55, bitrate: 350_000, crf: { vp9: 44, x264: 32, av1: 46 }, preview: { width: 240, frames: 16 } },
|
|
16
|
+
medium: { fps: 12, width: 1280, jpeg: 65, bitrate: 800_000, crf: { vp9: 38, x264: 29, av1: 40 }, preview: { width: 320, frames: 20 } },
|
|
17
|
+
high: { fps: 24, width: 1920, jpeg: 80, bitrate: 2_000_000, crf: { vp9: 32, x264: 24, av1: 34 }, preview: { width: 400, frames: 28 } }
|
|
18
|
+
}
|
|
19
|
+
/** Longest kept recording in seconds: a step keeps its last minute, a flow its last 15 minutes. */
|
|
20
|
+
export const MAX_SECONDS = { step: 60, flow: 900 }
|
|
21
|
+
export const preset = (q) => PRESETS[q] || PRESETS.medium
|
|
22
|
+
|
|
23
|
+
const execP = (bin, args, opts = {}) => new Promise((resolve, reject) => {
|
|
24
|
+
execFile(bin, args, { maxBuffer: 32 * 1024 * 1024, timeout: opts.timeout || 600000, encoding: opts.encoding || 'utf8' }, (error, stdout, stderr) => error ? reject(Object.assign(error, { stderr })) : resolve({ stdout, stderr }))
|
|
25
|
+
})
|
|
26
|
+
|
|
27
|
+
/** Install hints per tool and OS (macOS brew, Debian/Ubuntu apt, Windows winget or choco). */
|
|
28
|
+
export function installHint (tool, platform = process.platform) {
|
|
29
|
+
const H = {
|
|
30
|
+
ffmpeg: { darwin: 'brew install ffmpeg', linux: 'sudo apt install ffmpeg (or dnf install ffmpeg)', win32: 'winget install Gyan.FFmpeg (or choco install ffmpeg)' },
|
|
31
|
+
adb: { darwin: 'brew install --cask android-platform-tools', linux: 'sudo apt install adb', win32: 'winget install Google.PlatformTools (or choco install adb)' },
|
|
32
|
+
simctl: { darwin: 'Install Xcode with an iOS Simulator runtime (xcode-select --install for the command line tools)' }
|
|
33
|
+
}
|
|
34
|
+
return H[tool]?.[platform] || H[tool]?.linux || ''
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
let cached = null
|
|
38
|
+
/**
|
|
39
|
+
* What this machine can use for recordings: { ffmpeg: { path, version, encoders: { vp9, vp8, av1, h264, gif } } | null,
|
|
40
|
+
* adb, simctl, screencapture, platform }. `run` is stubbed in tests; KALOKO_FFMPEG points at another ffmpeg.
|
|
41
|
+
*/
|
|
42
|
+
export async function videoTools ({ run = execP, env = process.env, platform = process.platform, refresh = false } = {}) {
|
|
43
|
+
if (cached && !refresh && run === execP) return cached
|
|
44
|
+
const bin = env.KALOKO_FFMPEG || 'ffmpeg'
|
|
45
|
+
let ffmpeg = null
|
|
46
|
+
try {
|
|
47
|
+
const v = await run(bin, ['-hide_banner', '-version'], { timeout: 15000 })
|
|
48
|
+
const e = await run(bin, ['-hide_banner', '-encoders'], { timeout: 15000 })
|
|
49
|
+
const list = String(e.stdout)
|
|
50
|
+
const has = (name) => new RegExp(`^\\s*V\\S*\\s+${name}\\s`, 'm').test(list)
|
|
51
|
+
const h264 = ['libx264', platform === 'darwin' ? 'h264_videotoolbox' : null, platform === 'win32' ? 'h264_mf' : null, 'h264_nvenc', 'h264_vaapi'].filter(Boolean).find(has) || null
|
|
52
|
+
ffmpeg = {
|
|
53
|
+
path: bin,
|
|
54
|
+
version: (String(v.stdout).match(/ffmpeg version (\S+)/) || [])[1] || 'unknown',
|
|
55
|
+
encoders: { vp9: has('libvpx-vp9') ? 'libvpx-vp9' : null, vp8: has('libvpx') ? 'libvpx' : null, av1: ['libsvtav1', 'libaom-av1', 'librav1e'].find(has) || null, h264, gif: has('gif') ? 'gif' : null }
|
|
56
|
+
}
|
|
57
|
+
} catch {}
|
|
58
|
+
const ok = async (b, a) => { try { await run(b, a, { timeout: 15000 }); return true } catch { return false } }
|
|
59
|
+
const out = {
|
|
60
|
+
platform,
|
|
61
|
+
ffmpeg,
|
|
62
|
+
adb: await ok(env.KALOKO_ADB || 'adb', ['version']),
|
|
63
|
+
simctl: platform === 'darwin' ? await ok('xcrun', ['simctl', 'help']) : false,
|
|
64
|
+
screencapture: platform === 'darwin'
|
|
65
|
+
}
|
|
66
|
+
if (run === execP) cached = out
|
|
67
|
+
return out
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Formats a recording gets, from what the encoder page (WebCodecs, WebP) and ffmpeg can do. `codec`: auto | vp9 |
|
|
72
|
+
* av1 | h264; `mp4`: auto (when ffmpeg can make H.264) | true | false. Returns { webm, mp4, preview, notes }.
|
|
73
|
+
*/
|
|
74
|
+
export function chooseFormats ({ tools = {}, caps = {}, codec = 'auto', mp4 = 'auto' } = {}) {
|
|
75
|
+
const ff = tools.ffmpeg?.encoders || {}
|
|
76
|
+
const notes = []
|
|
77
|
+
let webm = null
|
|
78
|
+
if (codec === 'av1') {
|
|
79
|
+
// AV1 in WebM needs codec private data the browser does not hand out: ffmpeg encodes it
|
|
80
|
+
if (ff.av1) webm = { via: 'ffmpeg', codec: 'av1', encoder: ff.av1 }
|
|
81
|
+
else notes.push('AV1 needs ffmpeg with libsvtav1 or libaom; using VP9.')
|
|
82
|
+
}
|
|
83
|
+
if (!webm && codec !== 'h264') {
|
|
84
|
+
if (caps.vp9) webm = { via: 'webcodecs', codec: 'vp9' }
|
|
85
|
+
else if (caps.vp8) webm = { via: 'webcodecs', codec: 'vp8' }
|
|
86
|
+
else if (ff.vp9) webm = { via: 'ffmpeg', codec: 'vp9', encoder: ff.vp9 }
|
|
87
|
+
else if (ff.vp8) webm = { via: 'ffmpeg', codec: 'vp8', encoder: ff.vp8 }
|
|
88
|
+
}
|
|
89
|
+
let mp4Out = null
|
|
90
|
+
if (mp4 !== false && mp4 !== 'false' && ff.h264) mp4Out = { via: 'ffmpeg', encoder: ff.h264 }
|
|
91
|
+
else if ((mp4 === true || codec === 'h264') && !ff.h264) notes.push(`MP4 (H.264) needs ffmpeg: ${installHint('ffmpeg', tools.platform)}.`)
|
|
92
|
+
if (!webm && !mp4Out) notes.push(`No video encoder: this browser has no WebCodecs and ffmpeg is missing (${installHint('ffmpeg', tools.platform)}); only the poster and preview are kept.`)
|
|
93
|
+
const preview = caps.webp ? 'webp' : 'gif'
|
|
94
|
+
return { webm, mp4: mp4Out, preview, notes }
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/** Output size: the frame scaled to the preset's width, both sides even (encoders need it). */
|
|
98
|
+
export function outputSize (w, h, maxWidth) {
|
|
99
|
+
const k = Math.min(1, maxWidth / Math.max(1, w))
|
|
100
|
+
const even = (n) => Math.max(2, Math.round(n / 2) * 2)
|
|
101
|
+
return [even(w * k), even(h * k)]
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* The frames a clip encodes at a constant rate: for each tick the newest source frame at or before it, the pointer
|
|
106
|
+
* and the click ripples at that moment, and which ticks also make the preview and the poster.
|
|
107
|
+
* frames: [{ t }] sorted (ms); events: [{ t, type: move|down|up, x, y }] in frame pixels; labels: [{ t, text }].
|
|
108
|
+
* Returns { ticks, start, end, trimmed }; tick t is ms from the clip start.
|
|
109
|
+
*/
|
|
110
|
+
export function schedule ({ frames, events = [], labels = [], start, end, fps, maxSeconds, previewFrames = 20 }) {
|
|
111
|
+
if (!frames.length) return { ticks: [], start, end, trimmed: false }
|
|
112
|
+
start = Math.max(start ?? frames[0].t, frames[0].t)
|
|
113
|
+
end = Math.max(end ?? frames[frames.length - 1].t, start + 1000 / fps)
|
|
114
|
+
let trimmed = false
|
|
115
|
+
if (maxSeconds && end - start > maxSeconds * 1000) { start = end - maxSeconds * 1000; trimmed = true }
|
|
116
|
+
const step = 1000 / fps
|
|
117
|
+
const n = Math.max(1, Math.ceil((end - start) / step))
|
|
118
|
+
const ticks = []
|
|
119
|
+
let src = 0
|
|
120
|
+
// the source frame shown at the clip start may be older than the start (nothing repainted since)
|
|
121
|
+
while (src + 1 < frames.length && frames[src + 1].t <= start) src++
|
|
122
|
+
let ev = 0; let cursor = null
|
|
123
|
+
const pv = new Set(Array.from({ length: Math.min(previewFrames, n) }, (_, i) => Math.floor(i * n / Math.min(previewFrames, n))))
|
|
124
|
+
for (let i = 0; i < n; i++) {
|
|
125
|
+
const at = start + i * step
|
|
126
|
+
while (src + 1 < frames.length && frames[src + 1].t <= at) src++
|
|
127
|
+
while (ev < events.length && events[ev].t <= at) { if (events[ev].x != null) cursor = { x: events[ev].x, y: events[ev].y }; ev++ }
|
|
128
|
+
const rip = events.filter((e) => e.type === 'down' && e.t <= at && at - e.t < 450).map((e) => ({ x: e.x, y: e.y, a: (at - e.t) / 450 }))
|
|
129
|
+
const label = labels.filter((l) => l.t <= at && at - l.t < 1600).pop()?.text || null
|
|
130
|
+
ticks.push({ t: Math.round(i * step), src, ...(cursor ? { cur: cursor } : {}), ...(rip.length ? { rip } : {}), ...(label ? { label } : {}), ...(pv.has(i) ? { preview: true } : {}), ...(i === 0 ? { poster: true } : {}) })
|
|
131
|
+
}
|
|
132
|
+
return { ticks, start, end, trimmed }
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
// ---------- the encoder page (Chrome) ----------
|
|
136
|
+
|
|
137
|
+
/** Installs the encoder into a page (about:blank of the walk's browser or of an own headless Chrome). */
|
|
138
|
+
function install () {
|
|
139
|
+
if (window.__kenc) return
|
|
140
|
+
const CODECS = { vp9: 'vp09.00.10.08', vp8: 'vp8', h264: 'avc1.42001f' }
|
|
141
|
+
const b64 = (u8) => { let s = ''; for (let i = 0; i < u8.length; i += 0x8000) s += String.fromCharCode.apply(null, u8.subarray(i, i + 0x8000)); return btoa(s) }
|
|
142
|
+
const fromB64 = (s) => Uint8Array.from(atob(s), (c) => c.charCodeAt(0))
|
|
143
|
+
const blobBytes = async (canvas, type, q) => { const b = await canvas.convertToBlob({ type, quality: q }); return { type: b.type, data: b64(new Uint8Array(await b.arrayBuffer())) } }
|
|
144
|
+
const pointer = (g, x, y, s) => {
|
|
145
|
+
g.save(); g.translate(x, y); g.scale(s, s)
|
|
146
|
+
g.beginPath(); g.moveTo(0, 0); g.lineTo(0, 17); g.lineTo(4.5, 13); g.lineTo(7.5, 20); g.lineTo(10.5, 18.8); g.lineTo(7.6, 12); g.lineTo(13, 12); g.closePath()
|
|
147
|
+
g.fillStyle = '#151514'; g.fill(); g.lineWidth = 1.6; g.strokeStyle = '#ffffff'; g.stroke(); g.restore()
|
|
148
|
+
}
|
|
149
|
+
window.__kenc = {
|
|
150
|
+
async caps () {
|
|
151
|
+
const out = { webcodecs: typeof VideoEncoder !== 'undefined', vp9: false, vp8: false, h264: false, webp: false }
|
|
152
|
+
if (out.webcodecs) {
|
|
153
|
+
for (const [k, codec] of Object.entries(CODECS)) {
|
|
154
|
+
try { out[k] = !!(await VideoEncoder.isConfigSupported({ codec, width: 640, height: 480, bitrate: 500000, framerate: 12 })).supported } catch {}
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
try { const c = new OffscreenCanvas(2, 2); c.getContext('2d').fillRect(0, 0, 2, 2); out.webp = (await c.convertToBlob({ type: 'image/webp', quality: 0.5 })).type === 'image/webp' } catch {}
|
|
158
|
+
return out
|
|
159
|
+
},
|
|
160
|
+
start (cfg) {
|
|
161
|
+
this.cfg = cfg; this.chunks = []; this.bitmaps = new Map(); this.n = 0; this.error = null
|
|
162
|
+
this.canvas = new OffscreenCanvas(cfg.width, cfg.height); this.g = this.canvas.getContext('2d')
|
|
163
|
+
this.pcanvas = new OffscreenCanvas(cfg.pw, cfg.ph); this.pg = this.pcanvas.getContext('2d', { willReadFrequently: cfg.previewType === 'gif' })
|
|
164
|
+
this.enc = null
|
|
165
|
+
if (cfg.codec) {
|
|
166
|
+
this.enc = new VideoEncoder({ output: (chunk) => { const u = new Uint8Array(chunk.byteLength); chunk.copyTo(u); this.chunks.push({ ts: chunk.timestamp / 1000, key: chunk.type === 'key', data: b64(u) }) }, error: (e) => { this.error = String(e?.message || e) } })
|
|
167
|
+
this.enc.configure({ codec: CODECS[cfg.codec], width: cfg.width, height: cfg.height, bitrate: cfg.bitrate, framerate: cfg.fps, latencyMode: 'quality' })
|
|
168
|
+
}
|
|
169
|
+
return true
|
|
170
|
+
},
|
|
171
|
+
async feed ({ images, ticks }) {
|
|
172
|
+
const c = this.cfg; const g = this.g
|
|
173
|
+
for (const im of images) this.bitmaps.set(im.i, await createImageBitmap(new Blob([fromB64(im.b64)], { type: 'image/jpeg' })))
|
|
174
|
+
const out = { previews: [], poster: null }
|
|
175
|
+
for (const tk of ticks) {
|
|
176
|
+
for (const [k, bm] of this.bitmaps) if (k < tk.src) { bm.close(); this.bitmaps.delete(k) }
|
|
177
|
+
const bm = this.bitmaps.get(tk.src)
|
|
178
|
+
g.fillStyle = '#ffffff'; g.fillRect(0, 0, c.width, c.height)
|
|
179
|
+
if (bm) g.drawImage(bm, 0, 0, c.width, c.height)
|
|
180
|
+
if (tk.poster && !c.noPoster) out.poster = await blobBytes(this.canvas, 'image/webp', 0.8)
|
|
181
|
+
const s = c.width / c.srcWidth
|
|
182
|
+
for (const r of tk.rip || []) { g.beginPath(); g.arc(r.x * s, r.y * s, (8 + 26 * r.a) * c.ui, 0, Math.PI * 2); g.fillStyle = `rgba(255,228,82,${0.55 * (1 - r.a)})`; g.fill(); g.lineWidth = 2 * c.ui; g.strokeStyle = `rgba(21,21,20,${0.8 * (1 - r.a)})`; g.stroke() }
|
|
183
|
+
if (tk.cur) pointer(g, tk.cur.x * s, tk.cur.y * s, c.ui)
|
|
184
|
+
if (tk.label) {
|
|
185
|
+
g.font = `600 ${Math.round(15 * c.ui)}px system-ui, sans-serif`; const w = g.measureText(tk.label).width + 20 * c.ui
|
|
186
|
+
g.fillStyle = 'rgba(21,21,20,0.82)'; g.fillRect(12 * c.ui, 12 * c.ui, w, 30 * c.ui); g.fillStyle = '#ffe452'; g.textBaseline = 'middle'; g.fillText(tk.label, 22 * c.ui, 27 * c.ui)
|
|
187
|
+
}
|
|
188
|
+
if (this.enc) {
|
|
189
|
+
const f = new VideoFrame(this.canvas, { timestamp: Math.round(tk.t * 1000), duration: Math.round(1e6 / c.fps) })
|
|
190
|
+
this.enc.encode(f, { keyFrame: this.n % c.keyEvery === 0 }); f.close(); this.n++
|
|
191
|
+
while (this.enc.encodeQueueSize > 6) await new Promise((resolve) => setTimeout(resolve, 2))
|
|
192
|
+
}
|
|
193
|
+
if (tk.preview) {
|
|
194
|
+
this.pg.drawImage(this.canvas, 0, 0, c.pw, c.ph)
|
|
195
|
+
if (c.previewType === 'webp') out.previews.push(await blobBytes(this.pcanvas, 'image/webp', 0.62))
|
|
196
|
+
else out.previews.push({ type: 'rgba', data: b64(new Uint8Array(this.pg.getImageData(0, 0, c.pw, c.ph).data.buffer)) })
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
return { ...out, chunks: this.chunks.splice(0), error: this.error }
|
|
200
|
+
},
|
|
201
|
+
async finish () {
|
|
202
|
+
if (this.enc) { try { await this.enc.flush() } catch (e) { this.error = String(e?.message || e) } this.enc.close() }
|
|
203
|
+
for (const bm of this.bitmaps.values()) bm.close()
|
|
204
|
+
this.bitmaps.clear()
|
|
205
|
+
return { chunks: this.chunks.splice(0), error: this.error }
|
|
206
|
+
},
|
|
207
|
+
// a still image (title card of the walkthrough export): text on the brand colours, as PNG
|
|
208
|
+
async card ({ width, height, lines, verdict }) {
|
|
209
|
+
const cv = new OffscreenCanvas(width, height); const g = cv.getContext('2d'); const u = width / 1280
|
|
210
|
+
g.fillStyle = '#151514'; g.fillRect(0, 0, width, height)
|
|
211
|
+
const colors = { pass: '#3fb950', fail: '#f85149', partial: '#d29922', skip: '#8b949e' }
|
|
212
|
+
g.textBaseline = 'top'; let y = height * 0.36
|
|
213
|
+
lines.forEach((l, i) => { g.font = `${i === 1 ? 700 : 500} ${Math.round((i === 1 ? 44 : 22) * u)}px system-ui, sans-serif`; g.fillStyle = i === 1 ? '#ffffff' : '#c9c9c4'; g.fillText(l, 80 * u, y); y += (i === 1 ? 64 : 36) * u })
|
|
214
|
+
if (verdict) { g.font = `700 ${Math.round(22 * u)}px system-ui, sans-serif`; const w = g.measureText(verdict.text).width + 32 * u; g.fillStyle = colors[verdict.status] || '#ffe452'; g.fillRect(80 * u, y + 12 * u, w, 40 * u); g.fillStyle = '#151514'; g.fillText(verdict.text, 96 * u, y + 21 * u) }
|
|
215
|
+
g.fillStyle = '#ffe452'; g.fillRect(0, height - 8 * u, width, 8 * u)
|
|
216
|
+
return blobBytes(cv, 'image/png')
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
const NO_CAPS = { webcodecs: false, vp9: false, vp8: false, h264: false, webp: false }
|
|
222
|
+
const CAPS = new WeakMap()
|
|
223
|
+
/** WebCodecs needs a secure context: the encoder page opens a routed https page that never reaches the network. */
|
|
224
|
+
const ENCODER_URL = 'https://encoder.kaloko.invalid/'
|
|
225
|
+
/**
|
|
226
|
+
* Installs the encoder into `page` (moved to a secure origin first) and answers what it can do
|
|
227
|
+
* ({ webcodecs, vp9, vp8, h264, webp }); cached per page.
|
|
228
|
+
*/
|
|
229
|
+
export async function encoderCaps (page) {
|
|
230
|
+
if (!page) return NO_CAPS
|
|
231
|
+
if (CAPS.has(page)) return CAPS.get(page)
|
|
232
|
+
let caps = NO_CAPS
|
|
233
|
+
try {
|
|
234
|
+
if (!(await page.evaluate(() => window.isSecureContext && location.href !== 'about:blank'))) {
|
|
235
|
+
await page.route(ENCODER_URL, (route) => route.fulfill({ status: 200, contentType: 'text/html', body: '<!doctype html><title>encoder</title>' }))
|
|
236
|
+
await page.goto(ENCODER_URL)
|
|
237
|
+
}
|
|
238
|
+
await page.evaluate(install)
|
|
239
|
+
caps = await page.evaluate(() => window.__kenc.caps())
|
|
240
|
+
} catch {}
|
|
241
|
+
CAPS.set(page, caps)
|
|
242
|
+
return caps
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
/**
|
|
246
|
+
* Encodes a recorded clip. frames: [{ t, file, w, h }] (JPEG files, frame pixels); events and labels as in schedule().
|
|
247
|
+
* Writes <out>.webm / .mp4 / .webp (or .gif) preview / .poster.webp and returns what was written:
|
|
248
|
+
* { files: { webm, mp4, preview, poster }, duration_ms, width, height, codec, fps, trimmed, notes }.
|
|
249
|
+
*/
|
|
250
|
+
// the encoder page holds one clip at a time: parallel locales and named sessions take turns
|
|
251
|
+
const LOCKS = new WeakMap()
|
|
252
|
+
export function encodeClip (opts) {
|
|
253
|
+
if (!opts.page) return encodeClipNow(opts)
|
|
254
|
+
const prev = LOCKS.get(opts.page) || Promise.resolve()
|
|
255
|
+
const next = prev.catch(() => {}).then(() => encodeClipNow(opts))
|
|
256
|
+
LOCKS.set(opts.page, next.catch(() => {}))
|
|
257
|
+
return next
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
async function encodeClipNow ({ page, frames, events = [], labels = [], start, end, out, quality = 'medium', maxSeconds = MAX_SECONDS.step, codec = 'auto', mp4 = 'auto', tools = null, caps = null }) {
|
|
261
|
+
const P = preset(quality)
|
|
262
|
+
if (!frames?.length) return null
|
|
263
|
+
caps ||= await encoderCaps(page)
|
|
264
|
+
tools ||= await videoTools()
|
|
265
|
+
const formats = chooseFormats({ tools, caps, codec, mp4 })
|
|
266
|
+
const srcW = frames[0].w; const srcH = frames[0].h
|
|
267
|
+
const [width, height] = outputSize(srcW, srcH, P.width)
|
|
268
|
+
const [pw, ph] = outputSize(srcW, srcH, P.preview.width)
|
|
269
|
+
const sch = schedule({ frames, events, labels, start, end, fps: P.fps, maxSeconds, previewFrames: P.preview.frames })
|
|
270
|
+
const files = {}
|
|
271
|
+
const notes = [...formats.notes]
|
|
272
|
+
const duration = Math.round(sch.end - sch.start)
|
|
273
|
+
if (page) {
|
|
274
|
+
try {
|
|
275
|
+
const viaBrowser = formats.webm?.via === 'webcodecs' ? formats.webm.codec : null
|
|
276
|
+
await page.evaluate((cfg) => window.__kenc.start(cfg), { codec: viaBrowser, width, height, fps: P.fps, bitrate: P.bitrate, keyEvery: P.fps * 2, srcWidth: srcW, pw, ph, previewType: formats.preview, ui: Math.max(0.6, width / 1280) })
|
|
277
|
+
const chunks = []; const previews = []; let poster = null; let error = null
|
|
278
|
+
const sent = new Set()
|
|
279
|
+
for (let i = 0; i < sch.ticks.length; i += 24) {
|
|
280
|
+
const batch = sch.ticks.slice(i, i + 24)
|
|
281
|
+
const need = [...new Set(batch.map((t) => t.src))].filter((s) => !sent.has(s))
|
|
282
|
+
const images = []
|
|
283
|
+
for (const s of need) { sent.add(s); images.push({ i: s, b64: (await readFile(frames[s].file)).toString('base64') }) }
|
|
284
|
+
const r = await page.evaluate((arg) => window.__kenc.feed(arg), { images, ticks: batch })
|
|
285
|
+
chunks.push(...r.chunks); previews.push(...r.previews); poster ||= r.poster; error ||= r.error
|
|
286
|
+
}
|
|
287
|
+
const fin = await page.evaluate(() => window.__kenc.finish())
|
|
288
|
+
chunks.push(...fin.chunks); error ||= fin.error
|
|
289
|
+
if (poster) { files.poster = `${out}.poster.webp`; await writeFile(files.poster, Buffer.from(poster.data, 'base64')) }
|
|
290
|
+
if (previews.length) {
|
|
291
|
+
const ms = Math.max(90, Math.min(250, Math.round(Math.min(duration, 3500) / previews.length)))
|
|
292
|
+
if (formats.preview === 'webp' && previews[0].type === 'image/webp') { files.preview = `${out}.webp`; await writeFile(files.preview, animatedWebp(previews.map((p) => ({ webp: Buffer.from(p.data, 'base64'), ms })))) } else if (previews[0].type === 'rgba') { files.preview = `${out}.gif`; await writeFile(files.preview, encodeGif(previews.map((p) => ({ rgba: Buffer.from(p.data, 'base64'), ms })), { width: pw, height: ph })) }
|
|
293
|
+
}
|
|
294
|
+
if (viaBrowser) {
|
|
295
|
+
if (error || !chunks.length) notes.push(`The browser encoder failed${error ? `: ${error}` : ''}.`)
|
|
296
|
+
else { files.webm = `${out}.webm`; await writeFile(files.webm, muxWebm({ width, height, codec: viaBrowser, chunks: chunks.map((c) => ({ ...c, data: Buffer.from(c.data, 'base64') })), duration, fps: P.fps })) }
|
|
297
|
+
}
|
|
298
|
+
} catch (error) { notes.push(`The browser encoder failed: ${String(error.message).split('\n')[0]}`) }
|
|
299
|
+
}
|
|
300
|
+
// ffmpeg: WebM when the browser could not, AV1, and MP4 (H.264) next to the WebM for Safari
|
|
301
|
+
if (tools.ffmpeg && ((!files.webm && formats.webm?.via === 'ffmpeg') || formats.mp4)) {
|
|
302
|
+
const tmp = await mkdtemp(join(tmpdir(), 'kaloko-rec-'))
|
|
303
|
+
try {
|
|
304
|
+
let source = files.webm
|
|
305
|
+
if (!source) {
|
|
306
|
+
// the source frames at their own times (no pointer drawn: that needs the browser)
|
|
307
|
+
const first = sch.ticks[0]?.src || 0
|
|
308
|
+
const used = frames.slice(first).filter((f, k) => k === 0 || f.t <= sch.end)
|
|
309
|
+
const list = used.map((f, k) => `file '${f.file.replaceAll("'", "'\\''")}'\nduration ${Math.max(0.001, ((used[k + 1]?.t ?? sch.end) - Math.max(f.t, sch.start)) / 1000).toFixed(3)}`).join('\n') + `\nfile '${used[used.length - 1].file.replaceAll("'", "'\\''")}'\n`
|
|
310
|
+
await writeFile(join(tmp, 'list.txt'), list)
|
|
311
|
+
source = { concat: join(tmp, 'list.txt') }
|
|
312
|
+
}
|
|
313
|
+
if (!files.webm && formats.webm?.via === 'ffmpeg') {
|
|
314
|
+
files.webm = `${out}.webm`
|
|
315
|
+
await ffmpegEncode(tools, source, files.webm, { kind: formats.webm.codec, encoder: formats.webm.encoder, width, height, fps: P.fps, crf: P.crf[formats.webm.codec === 'av1' ? 'av1' : 'vp9'] })
|
|
316
|
+
}
|
|
317
|
+
if (formats.mp4) {
|
|
318
|
+
files.mp4 = `${out}.mp4`
|
|
319
|
+
await ffmpegEncode(tools, source, files.mp4, { kind: 'h264', encoder: formats.mp4.encoder, width, height, fps: P.fps, crf: P.crf.x264, bitrate: P.bitrate })
|
|
320
|
+
}
|
|
321
|
+
} catch (error) {
|
|
322
|
+
notes.push(`ffmpeg failed: ${String(error.stderr || error.message).trim().split('\n').pop()}`)
|
|
323
|
+
for (const k of ['mp4']) if (files[k]) { await rm(files[k], { force: true }); delete files[k] }
|
|
324
|
+
} finally { await rm(tmp, { recursive: true, force: true }) }
|
|
325
|
+
}
|
|
326
|
+
const bytes = {}
|
|
327
|
+
for (const [k, f] of Object.entries(files)) bytes[k] = (await stat(f).catch(() => ({ size: 0 }))).size
|
|
328
|
+
return { files, bytes, duration_ms: duration, width, height, codec: files.webm ? (formats.webm?.codec || null) : files.mp4 ? 'h264' : null, fps: P.fps, trimmed: sch.trimmed, start: sch.start, end: sch.end, notes }
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
/** ffmpeg arguments for one output; `source` is a file or { concat: list.txt } of still frames with durations. */
|
|
332
|
+
export function ffmpegArgs (source, outFile, { kind, encoder, width, height, fps, crf, bitrate }) {
|
|
333
|
+
const input = typeof source === 'string' ? ['-i', source] : ['-f', 'concat', '-safe', '0', '-i', source.concat]
|
|
334
|
+
const scale = `scale=${width}:${height}:force_original_aspect_ratio=decrease,pad=${width}:${height}:(ow-iw)/2:(oh-ih)/2:color=white,fps=${fps},format=yuv420p`
|
|
335
|
+
const codec = {
|
|
336
|
+
vp9: ['-c:v', encoder || 'libvpx-vp9', '-deadline', 'realtime', '-cpu-used', '8', '-row-mt', '1', '-b:v', '0', '-crf', String(crf)],
|
|
337
|
+
vp8: ['-c:v', encoder || 'libvpx', '-deadline', 'realtime', '-cpu-used', '8', '-b:v', String(bitrate || 800000), '-crf', String(Math.min(63, crf || 30))],
|
|
338
|
+
av1: encoder === 'libsvtav1' ? ['-c:v', 'libsvtav1', '-preset', '10', '-crf', String(crf)] : ['-c:v', encoder || 'libaom-av1', '-cpu-used', '8', '-row-mt', '1', '-b:v', '0', '-crf', String(crf)],
|
|
339
|
+
h264: encoder === 'libx264' ? ['-c:v', 'libx264', '-preset', 'veryfast', '-crf', String(crf), '-profile:v', 'high'] : ['-c:v', encoder, '-b:v', String(bitrate || 1000000)]
|
|
340
|
+
}[kind === 'vp8' ? 'vp8' : kind]
|
|
341
|
+
// a key frame every two seconds: chapters and the step markers seek without decoding from the start
|
|
342
|
+
return ['-y', '-v', 'error', ...input, '-an', '-vf', scale, ...codec, '-g', String(Math.max(1, Math.round(fps * 2))), ...(kind === 'h264' ? ['-movflags', '+faststart'] : []), outFile]
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
export async function ffmpegEncode (tools, source, outFile, opts) {
|
|
346
|
+
await execP(tools.ffmpeg.path, ffmpegArgs(source, outFile, opts), { timeout: 900000 })
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
/** Runs ffmpeg with raw arguments (export, native recordings). */
|
|
350
|
+
export const ffmpeg = (tools, args, timeout = 900000) => execP(tools.ffmpeg.path, ['-y', '-v', 'error', ...args], { timeout })
|
|
351
|
+
|
|
352
|
+
/** Duration of a media file in ms from ffmpeg's stderr; null when unknown. */
|
|
353
|
+
export async function mediaDuration (tools, file) {
|
|
354
|
+
try { await execP(tools.ffmpeg.path, ['-hide_banner', '-i', file], { timeout: 30000 }) } catch (e) {
|
|
355
|
+
const m = String(e.stderr || '').match(/Duration: (\d+):(\d+):(\d+(?:\.\d+)?)/)
|
|
356
|
+
if (m) return Math.round(((+m[1]) * 3600 + (+m[2]) * 60 + (+m[3])) * 1000)
|
|
357
|
+
}
|
|
358
|
+
return null
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
/**
|
|
362
|
+
* Poster and animated preview of a finished video file (native app recordings): frames come out of ffmpeg and are
|
|
363
|
+
* encoded in the encoder page (WebP) or as a GIF by ffmpeg itself. { poster, preview } paths or null each.
|
|
364
|
+
*/
|
|
365
|
+
export async function stillsFromVideo ({ tools, page, file, out, quality = 'medium', duration = null }) {
|
|
366
|
+
const P = preset(quality)
|
|
367
|
+
const res = { poster: null, preview: null }
|
|
368
|
+
if (!tools.ffmpeg) return res
|
|
369
|
+
const tmp = await mkdtemp(join(tmpdir(), 'kaloko-stills-'))
|
|
370
|
+
try {
|
|
371
|
+
const secs = Math.max(0.5, (duration || await mediaDuration(tools, file) || 3000) / 1000)
|
|
372
|
+
const rate = Math.max(0.2, P.preview.frames / secs)
|
|
373
|
+
await ffmpeg(tools, ['-i', file, '-vf', `fps=${rate.toFixed(3)},scale=${P.preview.width}:-2`, '-frames:v', String(P.preview.frames), join(tmp, 'f%03d.png')])
|
|
374
|
+
const names = (await readdir(tmp)).filter((f) => f.endsWith('.png')).sort()
|
|
375
|
+
if (!names.length) return res
|
|
376
|
+
await ffmpeg(tools, ['-i', file, '-frames:v', '1', '-vf', `scale='min(${P.width},iw)':-2`, `${out}.poster.png`]).catch(() => {})
|
|
377
|
+
const caps = await encoderCaps(page)
|
|
378
|
+
if (caps.webp) {
|
|
379
|
+
const frames = []
|
|
380
|
+
for (const n of names) frames.push({ webp: Buffer.from((await page.evaluate(async (b64) => { const bm = await createImageBitmap(new Blob([Uint8Array.from(atob(b64), (c) => c.charCodeAt(0))], { type: 'image/png' })); const c = new OffscreenCanvas(bm.width, bm.height); c.getContext('2d').drawImage(bm, 0, 0); const b = await c.convertToBlob({ type: 'image/webp', quality: 0.62 }); const u = new Uint8Array(await b.arrayBuffer()); let s = ''; for (let i = 0; i < u.length; i += 0x8000) s += String.fromCharCode.apply(null, u.subarray(i, i + 0x8000)); return btoa(s) }, (await readFile(join(tmp, n))).toString('base64'))), 'base64'), ms: Math.min(250, Math.max(90, Math.round(Math.min(secs * 1000, 3500) / names.length))) })
|
|
381
|
+
res.preview = `${out}.webp`; await writeFile(res.preview, animatedWebp(frames))
|
|
382
|
+
const posterPng = await readFile(`${out}.poster.png`).catch(() => null)
|
|
383
|
+
if (posterPng) {
|
|
384
|
+
const webp = await page.evaluate(async (b64) => { const bm = await createImageBitmap(new Blob([Uint8Array.from(atob(b64), (c) => c.charCodeAt(0))], { type: 'image/png' })); const c = new OffscreenCanvas(bm.width, bm.height); c.getContext('2d').drawImage(bm, 0, 0); const b = await c.convertToBlob({ type: 'image/webp', quality: 0.8 }); const u = new Uint8Array(await b.arrayBuffer()); let s = ''; for (let i = 0; i < u.length; i += 0x8000) s += String.fromCharCode.apply(null, u.subarray(i, i + 0x8000)); return btoa(s) }, posterPng.toString('base64'))
|
|
385
|
+
res.poster = `${out}.poster.webp`; await writeFile(res.poster, Buffer.from(webp, 'base64'))
|
|
386
|
+
}
|
|
387
|
+
} else if (tools.ffmpeg.encoders.gif) {
|
|
388
|
+
res.preview = `${out}.gif`
|
|
389
|
+
await ffmpeg(tools, ['-framerate', String(Math.max(4, Math.round(names.length / Math.min(secs, 3.5)))), '-i', join(tmp, 'f%03d.png'), '-vf', 'split[a][b];[a]palettegen=max_colors=128[p];[b][p]paletteuse=dither=bayer', '-loop', '0', res.preview])
|
|
390
|
+
}
|
|
391
|
+
await rm(`${out}.poster.png`, { force: true })
|
|
392
|
+
} catch {} finally { await rm(tmp, { recursive: true, force: true }) }
|
|
393
|
+
return res
|
|
394
|
+
}
|
|
395
|
+
|
|
396
|
+
/** Starts a long-running process (adb screenrecord, simctl recordVideo, screencapture) and returns it. */
|
|
397
|
+
export function startProcess (bin, args, opts = {}) {
|
|
398
|
+
const child = spawn(bin, args, { stdio: ['ignore', 'pipe', 'pipe'], ...opts })
|
|
399
|
+
let stderr = ''
|
|
400
|
+
child.stderr.on('data', (d) => { stderr += d; if (stderr.length > 20000) stderr = stderr.slice(-10000) })
|
|
401
|
+
child.output = () => stderr
|
|
402
|
+
child.done = new Promise((resolve) => child.on('close', (code, signal) => resolve({ code, signal, stderr })))
|
|
403
|
+
child.on('error', () => {})
|
|
404
|
+
return child
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
/**
|
|
408
|
+
* `kaloko doctor` lines about recordings: ffmpeg and its encoders, and the recorders the config's app environments
|
|
409
|
+
* need. Recording is optional, so a missing tool fails the check only when the config asks for what needs it.
|
|
410
|
+
*/
|
|
411
|
+
export function recordingChecks ({ config = {}, tools }) {
|
|
412
|
+
const out = []
|
|
413
|
+
const ff = tools.ffmpeg
|
|
414
|
+
const want = config.capture?.record
|
|
415
|
+
const needsFf = want && typeof want === 'object' && (want.mp4 === true || ['av1', 'h264'].includes(want.codec))
|
|
416
|
+
out.push({
|
|
417
|
+
name: 'recording',
|
|
418
|
+
ok: !!ff || !needsFf,
|
|
419
|
+
detail: ff ? `ffmpeg ${ff.version} · MP4 ${ff.encoders.h264 || '—'} · VP9 ${ff.encoders.vp9 ? 'yes' : '—'} · AV1 ${ff.encoders.av1 || '—'}` : 'no ffmpeg: web recordings are WebM from Chrome; MP4 for Safari, native app recordings and kaloko video export need ffmpeg',
|
|
420
|
+
hint: ff ? '' : `Install ffmpeg: ${installHint('ffmpeg', tools.platform)}`
|
|
421
|
+
})
|
|
422
|
+
const apps = new Set(Object.values(config.environments || {}).flatMap((e) => Object.keys(e?.app || {})))
|
|
423
|
+
if (apps.has('android')) out.push({ name: 'recording android', ok: !!tools.adb, detail: tools.adb ? 'adb screenrecord (3-minute segments, touches shown)' : 'adb not found', hint: tools.adb ? '' : installHint('adb', tools.platform) })
|
|
424
|
+
if (apps.has('ios')) out.push({ name: 'recording ios', ok: !!tools.simctl, detail: tools.simctl ? 'simctl io recordVideo (H.264)' : 'simctl not found', hint: tools.simctl ? '' : installHint('simctl', 'darwin') })
|
|
425
|
+
if (apps.has('desktop')) out.push({ name: 'recording desktop', ok: tools.platform === 'darwin', detail: tools.platform === 'darwin' ? `screencapture -v (Screen Recording permission: see "app … desktop" above)${ff ? '' : '; .mov until ffmpeg is installed'}` : 'desktop recording needs macOS', hint: '' })
|
|
426
|
+
return out
|
|
427
|
+
}
|