frameyard 0.7.0 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (334) hide show
  1. package/CHANGELOG.md +164 -7
  2. package/GLOSSARY.md +152 -18
  3. package/README.md +328 -285
  4. package/dist/assets/integrations.d.ts +8 -4
  5. package/dist/assets/integrations.d.ts.map +1 -1
  6. package/dist/assets/integrations.js +8 -4
  7. package/dist/assets/integrations.js.map +1 -1
  8. package/dist/assets/provenance.d.ts +2 -2
  9. package/dist/assets/provenance.d.ts.map +1 -1
  10. package/dist/assets/provenance.js +5 -3
  11. package/dist/assets/provenance.js.map +1 -1
  12. package/dist/assets/tts-supertonic.d.ts +28 -0
  13. package/dist/assets/tts-supertonic.d.ts.map +1 -0
  14. package/dist/assets/tts-supertonic.js +156 -0
  15. package/dist/assets/tts-supertonic.js.map +1 -0
  16. package/dist/assets/tts.d.ts +18 -6
  17. package/dist/assets/tts.d.ts.map +1 -1
  18. package/dist/assets/tts.js +219 -97
  19. package/dist/assets/tts.js.map +1 -1
  20. package/dist/cli.js +1705 -106
  21. package/dist/cli.js.map +1 -1
  22. package/dist/composition/descriptor.d.ts +17 -3
  23. package/dist/composition/descriptor.d.ts.map +1 -1
  24. package/dist/composition/descriptor.js +51 -28
  25. package/dist/composition/descriptor.js.map +1 -1
  26. package/dist/composition/format.d.ts +12 -0
  27. package/dist/composition/format.d.ts.map +1 -0
  28. package/dist/composition/format.js +28 -0
  29. package/dist/composition/format.js.map +1 -0
  30. package/dist/composition/index.d.ts +2 -1
  31. package/dist/composition/index.d.ts.map +1 -1
  32. package/dist/composition/index.js +1 -0
  33. package/dist/composition/index.js.map +1 -1
  34. package/dist/context.d.ts +82 -0
  35. package/dist/context.d.ts.map +1 -0
  36. package/dist/context.js +277 -0
  37. package/dist/context.js.map +1 -0
  38. package/dist/contracts.d.ts +80 -6
  39. package/dist/contracts.d.ts.map +1 -1
  40. package/dist/contracts.js +3 -0
  41. package/dist/contracts.js.map +1 -1
  42. package/dist/doctor.d.ts +14 -1
  43. package/dist/doctor.d.ts.map +1 -1
  44. package/dist/doctor.js +166 -7
  45. package/dist/doctor.js.map +1 -1
  46. package/dist/errors.d.ts +1 -1
  47. package/dist/errors.d.ts.map +1 -1
  48. package/dist/errors.js.map +1 -1
  49. package/dist/fonts/add.d.ts +41 -0
  50. package/dist/fonts/add.d.ts.map +1 -0
  51. package/dist/fonts/add.js +259 -0
  52. package/dist/fonts/add.js.map +1 -0
  53. package/dist/fonts/binary.d.ts +4 -0
  54. package/dist/fonts/binary.d.ts.map +1 -0
  55. package/dist/fonts/binary.js +21 -0
  56. package/dist/fonts/binary.js.map +1 -0
  57. package/dist/fonts/css.d.ts +4 -0
  58. package/dist/fonts/css.d.ts.map +1 -0
  59. package/dist/fonts/css.js +50 -0
  60. package/dist/fonts/css.js.map +1 -0
  61. package/dist/fonts/fontsource.d.ts +107 -0
  62. package/dist/fonts/fontsource.d.ts.map +1 -0
  63. package/dist/fonts/fontsource.js +565 -0
  64. package/dist/fonts/fontsource.js.map +1 -0
  65. package/dist/fonts/index.d.ts +11 -0
  66. package/dist/fonts/index.d.ts.map +1 -0
  67. package/dist/fonts/index.js +11 -0
  68. package/dist/fonts/index.js.map +1 -0
  69. package/dist/fonts/install.d.ts +24 -0
  70. package/dist/fonts/install.d.ts.map +1 -0
  71. package/dist/fonts/install.js +145 -0
  72. package/dist/fonts/install.js.map +1 -0
  73. package/dist/fonts/manifest.d.ts +21 -0
  74. package/dist/fonts/manifest.d.ts.map +1 -0
  75. package/dist/fonts/manifest.js +96 -0
  76. package/dist/fonts/manifest.js.map +1 -0
  77. package/dist/fonts/options.d.ts +9 -0
  78. package/dist/fonts/options.d.ts.map +1 -0
  79. package/dist/fonts/options.js +66 -0
  80. package/dist/fonts/options.js.map +1 -0
  81. package/dist/fonts/schema.d.ts +4 -0
  82. package/dist/fonts/schema.d.ts.map +1 -0
  83. package/dist/fonts/schema.js +316 -0
  84. package/dist/fonts/schema.js.map +1 -0
  85. package/dist/fonts/summary.d.ts +16 -0
  86. package/dist/fonts/summary.d.ts.map +1 -0
  87. package/dist/fonts/summary.js +22 -0
  88. package/dist/fonts/summary.js.map +1 -0
  89. package/dist/fonts/types.d.ts +56 -0
  90. package/dist/fonts/types.d.ts.map +1 -0
  91. package/dist/fonts/types.js +8 -0
  92. package/dist/fonts/types.js.map +1 -0
  93. package/dist/fonts/verify.d.ts +33 -0
  94. package/dist/fonts/verify.d.ts.map +1 -0
  95. package/dist/fonts/verify.js +200 -0
  96. package/dist/fonts/verify.js.map +1 -0
  97. package/dist/html/document-adapter.d.ts +3 -3
  98. package/dist/html/document-adapter.d.ts.map +1 -1
  99. package/dist/html/document-adapter.js +1 -1
  100. package/dist/html/document-adapter.js.map +1 -1
  101. package/dist/html/index.d.ts +1 -1
  102. package/dist/html/index.d.ts.map +1 -1
  103. package/dist/html/parse.d.ts +10 -2
  104. package/dist/html/parse.d.ts.map +1 -1
  105. package/dist/html/parse.js +14 -5
  106. package/dist/html/parse.js.map +1 -1
  107. package/dist/html/validate.d.ts +12 -1
  108. package/dist/html/validate.d.ts.map +1 -1
  109. package/dist/html/validate.js +292 -61
  110. package/dist/html/validate.js.map +1 -1
  111. package/dist/index.d.ts +4 -4
  112. package/dist/index.d.ts.map +1 -1
  113. package/dist/index.js +2 -2
  114. package/dist/index.js.map +1 -1
  115. package/dist/inspect.d.ts +61 -0
  116. package/dist/inspect.d.ts.map +1 -0
  117. package/dist/inspect.js +325 -0
  118. package/dist/inspect.js.map +1 -0
  119. package/dist/lint.d.ts +63 -0
  120. package/dist/lint.d.ts.map +1 -0
  121. package/dist/lint.js +265 -0
  122. package/dist/lint.js.map +1 -0
  123. package/dist/policy.d.ts +69 -0
  124. package/dist/policy.d.ts.map +1 -0
  125. package/dist/policy.js +361 -0
  126. package/dist/policy.js.map +1 -0
  127. package/dist/preview/main.d.ts +2 -2
  128. package/dist/preview/main.d.ts.map +1 -1
  129. package/dist/preview/main.js +35 -14
  130. package/dist/preview/main.js.map +1 -1
  131. package/dist/preview/timeline-data.d.ts +17 -65
  132. package/dist/preview/timeline-data.d.ts.map +1 -1
  133. package/dist/preview/timeline-host.d.ts.map +1 -1
  134. package/dist/preview/timeline-host.js +73 -38
  135. package/dist/preview/timeline-host.js.map +1 -1
  136. package/dist/preview/timeline-main.d.ts.map +1 -1
  137. package/dist/preview/timeline-main.js +47 -3
  138. package/dist/preview/timeline-main.js.map +1 -1
  139. package/dist/project.d.ts +13 -8
  140. package/dist/project.d.ts.map +1 -1
  141. package/dist/project.js +20 -17
  142. package/dist/project.js.map +1 -1
  143. package/dist/renderer/browser.d.ts +2 -2
  144. package/dist/renderer/browser.d.ts.map +1 -1
  145. package/dist/renderer/browser.js +10 -4
  146. package/dist/renderer/browser.js.map +1 -1
  147. package/dist/renderer/ffmpeg.d.ts +44 -1
  148. package/dist/renderer/ffmpeg.d.ts.map +1 -1
  149. package/dist/renderer/ffmpeg.js +371 -73
  150. package/dist/renderer/ffmpeg.js.map +1 -1
  151. package/dist/renderer/index.d.ts +40 -4
  152. package/dist/renderer/index.d.ts.map +1 -1
  153. package/dist/renderer/index.js +100 -25
  154. package/dist/renderer/index.js.map +1 -1
  155. package/dist/renderer/keyframe-expressions.d.ts +39 -0
  156. package/dist/renderer/keyframe-expressions.d.ts.map +1 -0
  157. package/dist/renderer/keyframe-expressions.js +180 -0
  158. package/dist/renderer/keyframe-expressions.js.map +1 -0
  159. package/dist/renderer/timeline.d.ts +2 -0
  160. package/dist/renderer/timeline.d.ts.map +1 -1
  161. package/dist/renderer/timeline.js +168 -106
  162. package/dist/renderer/timeline.js.map +1 -1
  163. package/dist/runtime/host.d.ts.map +1 -1
  164. package/dist/runtime/host.js +159 -31
  165. package/dist/runtime/host.js.map +1 -1
  166. package/dist/scaffold.d.ts +3 -0
  167. package/dist/scaffold.d.ts.map +1 -1
  168. package/dist/scaffold.js +48 -317
  169. package/dist/scaffold.js.map +1 -1
  170. package/dist/server/composition-html.d.ts +15 -7
  171. package/dist/server/composition-html.d.ts.map +1 -1
  172. package/dist/server/composition-html.js +72 -16
  173. package/dist/server/composition-html.js.map +1 -1
  174. package/dist/server/server.d.ts.map +1 -1
  175. package/dist/server/server.js +56 -6
  176. package/dist/server/server.js.map +1 -1
  177. package/dist/server/timeline-preview-data.d.ts.map +1 -1
  178. package/dist/server/timeline-preview-data.js +12 -23
  179. package/dist/server/timeline-preview-data.js.map +1 -1
  180. package/dist/server/timeline-preview-html.d.ts.map +1 -1
  181. package/dist/server/timeline-preview-html.js +20 -3
  182. package/dist/server/timeline-preview-html.js.map +1 -1
  183. package/dist/templates.d.ts +16 -0
  184. package/dist/templates.d.ts.map +1 -0
  185. package/dist/templates.js +186 -0
  186. package/dist/templates.js.map +1 -0
  187. package/dist/timeline/easing.d.ts +36 -0
  188. package/dist/timeline/easing.d.ts.map +1 -0
  189. package/dist/timeline/easing.js +225 -0
  190. package/dist/timeline/easing.js.map +1 -0
  191. package/dist/timeline/keyframes.d.ts +23 -0
  192. package/dist/timeline/keyframes.d.ts.map +1 -0
  193. package/dist/timeline/keyframes.js +95 -0
  194. package/dist/timeline/keyframes.js.map +1 -0
  195. package/dist/timeline/plan.d.ts +6 -12
  196. package/dist/timeline/plan.d.ts.map +1 -1
  197. package/dist/timeline/plan.js +17 -30
  198. package/dist/timeline/plan.js.map +1 -1
  199. package/dist/timeline/resolve.d.ts +16 -3
  200. package/dist/timeline/resolve.d.ts.map +1 -1
  201. package/dist/timeline/resolve.js +119 -13
  202. package/dist/timeline/resolve.js.map +1 -1
  203. package/dist/timeline/scene.d.ts +126 -0
  204. package/dist/timeline/scene.d.ts.map +1 -0
  205. package/dist/timeline/scene.js +109 -0
  206. package/dist/timeline/scene.js.map +1 -0
  207. package/dist/timeline/schema.d.ts +38 -0
  208. package/dist/timeline/schema.d.ts.map +1 -1
  209. package/dist/timeline/schema.js +290 -9
  210. package/dist/timeline/schema.js.map +1 -1
  211. package/dist/timeline/transitions.d.ts +21 -0
  212. package/dist/timeline/transitions.d.ts.map +1 -0
  213. package/dist/timeline/transitions.js +23 -0
  214. package/dist/timeline/transitions.js.map +1 -0
  215. package/dist/version.d.ts +1 -1
  216. package/dist/version.js +1 -1
  217. package/docs/adr/0007-make-html-documents-the-composition-unit.md +2 -2
  218. package/docs/adr/0008-share-the-composition-document-grammar.md +1 -1
  219. package/docs/adr/0010-make-timelines-the-render-unit.md +3 -2
  220. package/docs/adr/0011-compose-timeline-frames-in-a-live-browser-host.md +1 -1
  221. package/docs/adr/0012-render-timelines-with-segmented-lossless-clip-intermediates.md +1 -1
  222. package/docs/adr/0014-compose-timeline-clips-with-css-blend-modes.md +38 -0
  223. package/docs/adr/0015-inherit-composition-format-from-the-project.md +28 -0
  224. package/docs/adr/0016-author-compositions-as-html-fragments.md +32 -0
  225. package/docs/adr/0017-render-timeless-compositions-as-stills.md +99 -0
  226. package/docs/adr/0018-animate-timeline-clips-with-keyframes.md +35 -0
  227. package/docs/adr/0019-make-same-track-overlap-the-transition-window.md +36 -0
  228. package/docs/adr/0020-crossfade-audio-clips-on-audio-tracks.md +27 -0
  229. package/docs/adr/0021-project-timeline-scenes-once.md +32 -0
  230. package/docs/adr/0022-gate-heavy-operations-with-a-workspace-resource-policy.md +35 -0
  231. package/docs/adr/0023-pin-fonts-as-project-assets.md +98 -0
  232. package/docs/migration-0.4.md +5 -3
  233. package/docs/migration-0.8.md +163 -0
  234. package/docs/preview-testing.md +23 -3
  235. package/docs/{v0.1-design.md → v0.x-design.md} +86 -53
  236. package/examples/blend/README.md +106 -0
  237. package/examples/blend/assets/ink.mp4 +0 -0
  238. package/examples/blend/compositions/aurora.html +55 -0
  239. package/examples/blend/compositions/backdrop.html +74 -0
  240. package/examples/blend/compositions/labels.html +149 -0
  241. package/examples/blend/fy.config.js +11 -0
  242. package/examples/blend/scripts/make-ink.sh +26 -0
  243. package/examples/blend/timelines/blend.timeline.json +155 -0
  244. package/examples/demo/README.md +2 -2
  245. package/examples/demo/compositions/demo.html +243 -251
  246. package/examples/intro/README.md +71 -0
  247. package/examples/intro/assets/fonts/figtree/LICENSE.txt +93 -0
  248. package/examples/intro/assets/fonts/figtree/figtree-latin-wght-normal.woff2 +0 -0
  249. package/examples/intro/assets/fonts/instrument-serif/LICENSE.txt +93 -0
  250. package/examples/intro/assets/fonts/instrument-serif/instrument-serif-latin-400-italic.woff2 +0 -0
  251. package/examples/intro/assets/fonts/space-mono/LICENSE.txt +93 -0
  252. package/examples/intro/assets/fonts/space-mono/space-mono-latin-400-normal.woff2 +0 -0
  253. package/examples/intro/assets/fonts/space-mono/space-mono-latin-700-normal.woff2 +0 -0
  254. package/examples/intro/assets/grain.png +0 -0
  255. package/examples/intro/assets/score.ogg +0 -0
  256. package/examples/intro/compositions/brand.html +412 -0
  257. package/examples/intro/compositions/chrome.controller.js +63 -0
  258. package/examples/intro/compositions/chrome.html +256 -0
  259. package/examples/intro/compositions/lattice.controller.js +148 -0
  260. package/examples/intro/compositions/lattice.html +308 -0
  261. package/examples/intro/compositions/opening.html +504 -0
  262. package/examples/intro/compositions/studio.html +930 -0
  263. package/examples/intro/compositions/theme.css +218 -0
  264. package/examples/intro/fy.config.js +13 -0
  265. package/examples/intro/fy.fonts.json +100 -0
  266. package/examples/intro/scripts/make-grain.mjs +79 -0
  267. package/examples/intro/scripts/make-score.mjs +702 -0
  268. package/examples/intro/timelines/intro.timeline.json +60 -0
  269. package/examples/keyframes/README.md +76 -0
  270. package/examples/keyframes/compositions/backdrop.html +60 -0
  271. package/examples/keyframes/compositions/card.html +99 -0
  272. package/examples/keyframes/fy.config.js +10 -0
  273. package/examples/keyframes/timelines/keyframes.timeline.json +60 -0
  274. package/examples/nested/README.md +112 -0
  275. package/examples/nested/assets/pulse.wav +0 -0
  276. package/examples/nested/compositions/backdrop.html +84 -0
  277. package/examples/nested/compositions/card.html +275 -0
  278. package/examples/nested/compositions/overlay.html +307 -0
  279. package/examples/nested/fy.config.js +13 -0
  280. package/examples/nested/scripts/make-pulse.sh +20 -0
  281. package/examples/nested/timelines/chapter-a.timeline.json +39 -0
  282. package/examples/nested/timelines/chapter-b.timeline.json +39 -0
  283. package/examples/nested/timelines/nested-reel.timeline.json +52 -0
  284. package/examples/static/README.md +17 -0
  285. package/examples/static/compositions/still.html +37 -0
  286. package/examples/static/fy.config.js +9 -0
  287. package/examples/static/timelines/still.timeline.json +13 -0
  288. package/examples/transitions/README.md +128 -0
  289. package/examples/transitions/assets/alpha.wav +0 -0
  290. package/examples/transitions/assets/beta.wav +0 -0
  291. package/examples/transitions/assets/gamma.wav +0 -0
  292. package/examples/transitions/compositions/alpha.html +56 -0
  293. package/examples/transitions/compositions/beta.html +56 -0
  294. package/examples/transitions/compositions/gamma.html +56 -0
  295. package/examples/transitions/fy.config.js +12 -0
  296. package/examples/transitions/timelines/audio-crossfade.timeline.json +44 -0
  297. package/examples/transitions/timelines/transitions.timeline.json +31 -0
  298. package/examples/video/README.md +4 -4
  299. package/examples/video/compositions/video-mp4.html +349 -357
  300. package/examples/video/compositions/video-overlay.html +337 -345
  301. package/examples/video/compositions/video.html +349 -357
  302. package/examples/webgl/README.md +1 -1
  303. package/examples/webgl/compositions/webgl.html +205 -213
  304. package/package.json +13 -5
  305. package/scaffolds/init/AGENTS.md +131 -0
  306. package/scaffolds/init/README.md +56 -0
  307. package/scaffolds/init/compositions/hello.html +79 -0
  308. package/scaffolds/init/fy.config.js +9 -0
  309. package/scaffolds/init/gitignore.template +1 -0
  310. package/scaffolds/init/package.json +11 -0
  311. package/scaffolds/init/props/hello.json +4 -0
  312. package/scaffolds/init/props/main.json +6 -0
  313. package/scaffolds/init/timelines/main.timeline.json +20 -0
  314. package/scaffolds/local-models/LOCAL-MODELS.md +77 -0
  315. package/scaffolds/local-models/fy.policy.json +5 -0
  316. package/scaffolds/local-models/opencode/opencode.jsonc +17 -0
  317. package/scaffolds/local-models/pi/APPEND_SYSTEM.md +21 -0
  318. package/scaffolds/local-models/pi/sandbox.json +15 -0
  319. package/scaffolds/local-models/pi/settings.json +15 -0
  320. package/skills/frameyard/README.md +1 -0
  321. package/skills/frameyard/SKILL.md +59 -59
  322. package/skills/frameyard/references/audio-assets.md +32 -3
  323. package/skills/frameyard/references/common-mistakes.md +81 -7
  324. package/skills/frameyard/references/controllers.md +5 -4
  325. package/skills/frameyard/references/deterministic-authoring.md +49 -37
  326. package/skills/frameyard/references/diagnostics.md +42 -4
  327. package/skills/frameyard/references/failure-recovery.md +56 -10
  328. package/skills/frameyard/references/fonts.md +132 -0
  329. package/skills/frameyard/references/json-results.md +165 -10
  330. package/skills/frameyard/references/quick-reference.md +222 -0
  331. package/skills/frameyard/references/timelines.md +109 -15
  332. /package/examples/demo/{video.config.js → fy.config.js} +0 -0
  333. /package/examples/video/{video.config.js → fy.config.js} +0 -0
  334. /package/examples/webgl/{video.config.js → fy.config.js} +0 -0
@@ -1,6 +1,9 @@
1
1
  import { spawn } from "node:child_process";
2
2
  import { writeFile } from "node:fs/promises";
3
3
  import { FrameyardError } from "../errors.js";
4
+ import { evaluatePresentation } from "../timeline/keyframes.js";
5
+ import { transitionProgress } from "../timeline/transitions.js";
6
+ import { keyframeChannelExpression, transitionGainExpression, transitionProgressExpression, } from "./keyframe-expressions.js";
4
7
  const STDERR_LIMIT = 16 * 1024;
5
8
  function captureTail(child) {
6
9
  let tail = "";
@@ -302,8 +305,7 @@ export async function probeMedia(filePath, options = {}) {
302
305
  throw new FrameyardError("FFPROBE_FAILED", `Could not parse ffprobe output for ${filePath}.`, { cause: error });
303
306
  }
304
307
  }
305
- /** Normalizes any decodable source to the 16 kHz mono PCM WAV that whisper.cpp expects. */
306
- export async function normalizeAudioForTranscription(options) {
308
+ async function normalizeAudio(options) {
307
309
  await runCaptured("ffmpeg", [
308
310
  "-hide_banner",
309
311
  "-loglevel",
@@ -318,12 +320,20 @@ export async function normalizeAudioForTranscription(options) {
318
320
  "-ac",
319
321
  "1",
320
322
  "-ar",
321
- "16000",
323
+ String(options.sampleRate),
322
324
  "-c:a",
323
325
  "pcm_s16le",
324
326
  options.outputPath,
325
327
  ], { signal: options.signal });
326
328
  }
329
+ /** Normalizes any decodable source to the 16 kHz mono PCM WAV that whisper.cpp expects. */
330
+ export async function normalizeAudioForTranscription(options) {
331
+ await normalizeAudio({ ...options, sampleRate: 16_000 });
332
+ }
333
+ /** Normalizes generated speech to the 24 kHz mono PCM WAV that Frameyard publishes. */
334
+ export async function normalizeSpeechAudio(options) {
335
+ await normalizeAudio({ ...options, sampleRate: 24_000 });
336
+ }
327
337
  export async function mixAudioTimeline(options) {
328
338
  if (options.clips.length === 0) {
329
339
  throw new FrameyardError("AUDIO_INVALID", "Cannot mix an empty audio timeline.");
@@ -340,10 +350,22 @@ export async function mixAudioTimeline(options) {
340
350
  trim.push(`duration=${formatNumber(clip.trimDuration)}`);
341
351
  }
342
352
  const delayMs = Math.round((clip.startFrame * 1000) / options.fps);
353
+ const gain = clip.volumeEnvelopes?.length
354
+ ? clip.volumeEnvelopes
355
+ .map((envelope) => transitionGainExpression({
356
+ progressExpression: `((${formatNumber(clip.startFrame)}+t*${formatNumber(options.fps)}-${formatNumber(envelope.startFrame)})/${formatNumber(envelope.endFrame - envelope.startFrame)})`,
357
+ ...(envelope.easing !== undefined ? { easing: envelope.easing } : {}),
358
+ role: envelope.role,
359
+ }))
360
+ .join("*")
361
+ : undefined;
362
+ const volume = gain === undefined
363
+ ? `volume=${formatNumber(clip.volume)}`
364
+ : `volume='${formatNumber(clip.volume)}*(${gain})':eval=frame`;
343
365
  const chain = [
344
366
  `atrim=${trim.join(":")}`,
345
367
  "asetpts=PTS-STARTPTS",
346
- `volume=${formatNumber(clip.volume)}`,
368
+ volume,
347
369
  "aresample=48000",
348
370
  "aformat=sample_fmts=fltp:channel_layouts=stereo",
349
371
  "apad",
@@ -492,6 +514,44 @@ export async function startVideoEncoder(options) {
492
514
  abort,
493
515
  };
494
516
  }
517
+ /**
518
+ * Builds FFmpeg's narrow layer input from a projected scene layer. The mapper
519
+ * owns the scene-to-filter boundary; capture chooses the artifact and its input
520
+ * mode, while the scene owns presentation, keyframes, and transition roles.
521
+ */
522
+ export function sceneLayerToCompositeLayer(layer, artifact) {
523
+ const transform = layer.transform;
524
+ const transition = layer.transition;
525
+ return {
526
+ input: artifact.input,
527
+ path: artifact.path,
528
+ sourceWidth: layer.sourceWidth,
529
+ sourceHeight: layer.sourceHeight,
530
+ ...(transform.width !== undefined && transform.height !== undefined
531
+ ? { width: transform.width, height: transform.height }
532
+ : {}),
533
+ fit: transform.fit,
534
+ x: transform.x,
535
+ y: transform.y,
536
+ scale: transform.scale,
537
+ rotation: transform.rotation,
538
+ opacity: transform.opacity,
539
+ blend: layer.blend,
540
+ ...(artifact.sourceStartFrame !== undefined ? { sourceStartFrame: artifact.sourceStartFrame } : {}),
541
+ ...(layer.keyframes !== undefined ? { keyframes: layer.keyframes } : {}),
542
+ clipStart: layer.clip.start,
543
+ ...(transition !== undefined
544
+ ? {
545
+ transition: {
546
+ role: transition.role,
547
+ start: transition.window.start,
548
+ end: transition.window.end,
549
+ ...(transition.window.easing !== undefined ? { easing: transition.window.easing } : {}),
550
+ },
551
+ }
552
+ : {}),
553
+ };
554
+ }
495
555
  function ffmpegBackgroundColor(value) {
496
556
  if (value === "transparent") {
497
557
  return "black@0.0";
@@ -533,47 +593,206 @@ function fitFilter(layer, stageWidth, stageHeight) {
533
593
  const fittedHeight = Math.max(1, Math.round(sourceHeight * scale));
534
594
  return `scale=${fittedWidth}:${fittedHeight},pad=${width}:${height}:(ow-iw)/2:(oh-ih)/2:color=black@0`;
535
595
  }
536
- function layerFilter(layer, stageWidth, stageHeight, fps) {
537
- const filters = [
596
+ function layerTransform(layer) {
597
+ return {
598
+ fit: layer.fit,
599
+ x: layer.x,
600
+ y: layer.y,
601
+ ...(layer.width !== undefined && layer.height !== undefined ? { width: layer.width, height: layer.height } : {}),
602
+ scale: layer.scale,
603
+ rotation: layer.rotation,
604
+ opacity: layer.opacity,
605
+ };
606
+ }
607
+ function hasKeyframe(layer, channel) {
608
+ return layer.keyframes?.[channel] !== undefined;
609
+ }
610
+ /** The incoming layer's animated alpha at an absolute timeline frame. */
611
+ function transitionAlphaAt(layer, frame) {
612
+ if (layer.transition?.role !== "in") {
613
+ return 1;
614
+ }
615
+ return transitionProgress(layer.transition, frame);
616
+ }
617
+ /** False when a keyed or transitioning layer is invisible for every frame of the range. */
618
+ function isLayerVisibleInRange(layer, frameOffset, frames) {
619
+ const animated = hasKeyframe(layer, "opacity") || hasKeyframe(layer, "scale");
620
+ if (!animated && layer.transition?.role !== "in") {
621
+ return layer.opacity > 0 && layer.scale > 0;
622
+ }
623
+ const transform = layerTransform(layer);
624
+ const clipStart = layer.clipStart ?? 0;
625
+ for (let index = 0; index < frames; index += 1) {
626
+ const frame = frameOffset + index;
627
+ const values = evaluatePresentation(transform, layer.keyframes, frame - clipStart);
628
+ if (values.opacity * transitionAlphaAt(layer, frame) > 0 && values.scale > 0) {
629
+ return true;
630
+ }
631
+ }
632
+ return false;
633
+ }
634
+ function baseLayerFilters(layer, stageWidth, stageHeight, fps) {
635
+ return [
538
636
  // Matroska and WebM timestamps can be quantized to milliseconds. Normalize
539
637
  // every timed layer to the project frame rate before overlay framesync so
540
638
  // it does not duplicate or drop frames. Still images are already static.
541
- ...(layer.kind === "image" ? [] : [`fps=${formatNumber(fps)}`]),
639
+ ...(layer.input === "image" ? [] : [`fps=${formatNumber(fps)}`]),
542
640
  fitFilter(layer, stageWidth, stageHeight),
543
641
  "format=rgba",
544
642
  ];
545
- if (layer.scale !== 1) {
546
- filters.push(`scale=iw*${formatNumber(layer.scale)}:ih*${formatNumber(layer.scale)}`);
643
+ }
644
+ /** Dimensions entering the rotate filter for this layer. */
645
+ function rotationInputSize(layer, stageWidth, stageHeight) {
646
+ const width = layer.width ?? stageWidth;
647
+ const height = layer.height ?? stageHeight;
648
+ if (layer.keyframes?.scale !== undefined) {
649
+ // Animated scale is applied after rotation, keeping the rotate input fixed.
650
+ return { width, height };
651
+ }
652
+ return {
653
+ width: Math.max(1, Math.ceil(width * layer.scale)),
654
+ height: Math.max(1, Math.ceil(height * layer.scale)),
655
+ };
656
+ }
657
+ function rotatedBounds(width, height, degrees) {
658
+ const radians = (degrees * Math.PI) / 180;
659
+ const cosine = Math.abs(Math.cos(radians));
660
+ const sine = Math.abs(Math.sin(radians));
661
+ // Two extra pixels absorb filter rounding and anti-aliased edge pixels.
662
+ return {
663
+ width: Math.max(1, Math.ceil(width * cosine + height * sine) + 2),
664
+ height: Math.max(1, Math.ceil(width * sine + height * cosine) + 2),
665
+ };
666
+ }
667
+ /**
668
+ * FFmpeg's rotate filter clips to a fixed output size. A keyed rotation uses a
669
+ * square diagonal bound so every segment uses the same size and corners stay
670
+ * visible at any angle. Static rotation uses its exact bounding box.
671
+ */
672
+ function rotationOutputSize(layer, stageWidth, stageHeight) {
673
+ const input = rotationInputSize(layer, stageWidth, stageHeight);
674
+ if (layer.keyframes?.rotation) {
675
+ const diagonal = Math.max(1, Math.ceil(Math.hypot(input.width, input.height)) + 2);
676
+ return { width: diagonal, height: diagonal };
677
+ }
678
+ return rotatedBounds(input.width, input.height, layer.rotation);
679
+ }
680
+ function needsOpacityMask(layer) {
681
+ return hasKeyframe(layer, "opacity") || hasKeyframe(layer, "scale") || layer.transition?.role === "in";
682
+ }
683
+ /**
684
+ * Presentation filters after the optional opacity mask. Animated scale is
685
+ * applied after rotation because uniform scale and rotation commute; rotation
686
+ * then keeps a fixed input size for the rotate filter.
687
+ */
688
+ function presentationLayerFilters(layer, stageWidth, stageHeight, frameOffset) {
689
+ const filters = [];
690
+ const clipStart = layer.clipStart ?? 0;
691
+ const frameOptions = { frameVariable: "n", frameOffset, clipStart };
692
+ const scaleKeys = layer.keyframes?.scale;
693
+ const rotationKeys = layer.keyframes?.rotation;
694
+ const opacityKeys = layer.keyframes?.opacity;
695
+ const addRotation = () => {
696
+ if (rotationKeys || layer.rotation !== 0) {
697
+ const output = rotationOutputSize(layer, stageWidth, stageHeight);
698
+ const outputSize = `ow=${output.width}:oh=${output.height}`;
699
+ if (rotationKeys) {
700
+ const angle = keyframeChannelExpression("rotation", rotationKeys, frameOptions);
701
+ filters.push(`rotate=angle='(${angle})*PI/180':fillcolor=black@0:${outputSize}`);
702
+ }
703
+ else {
704
+ filters.push(`rotate=angle=${formatNumber(layer.rotation)}*PI/180:fillcolor=black@0:${outputSize}`);
705
+ }
706
+ }
707
+ };
708
+ if (scaleKeys) {
709
+ addRotation();
710
+ const scale = keyframeChannelExpression("scale", scaleKeys, frameOptions);
711
+ filters.push(`scale=w='max(1,iw*(${scale}))':h='max(1,ih*(${scale}))':eval=frame`);
547
712
  }
548
- if (layer.rotation !== 0) {
549
- filters.push(`rotate=angle=${formatNumber(layer.rotation)}*PI/180:fillcolor=black@0`);
713
+ else {
714
+ if (layer.scale !== 1) {
715
+ filters.push(`scale=iw*${formatNumber(layer.scale)}:ih*${formatNumber(layer.scale)}`);
716
+ }
717
+ addRotation();
550
718
  }
551
- if (layer.opacity < 1) {
719
+ // A transitioning layer carries its opacity in the animated mask instead.
720
+ if (!opacityKeys && !scaleKeys && layer.transition?.role !== "in" && layer.opacity < 1) {
552
721
  filters.push(`colorchannelmixer=aa=${formatNumber(layer.opacity)}`);
553
722
  }
554
- return filters.join(",");
723
+ return filters;
555
724
  }
556
- /** Encode a numbered PNG sequence into a lossless alpha-preserving FFV1 file. */
557
- export async function encodePngSequenceToFfv1(options) {
558
- await runCaptured("ffmpeg", [
559
- "-y",
560
- "-v",
561
- "error",
562
- "-framerate",
563
- formatNumber(options.fps),
564
- "-i",
565
- options.pattern,
566
- "-c:v",
567
- "ffv1",
568
- "-level",
569
- "3",
570
- "-pix_fmt",
571
- "bgra",
572
- options.outputPath,
573
- ], { signal: options.signal });
725
+ /** Builds the animated alpha mask at the fitted box size. */
726
+ function opacityMaskFilter(options) {
727
+ const { layer, stageWidth, stageHeight, frameOffset, inputIndex, outputLabel } = options;
728
+ const clipStart = layer.clipStart ?? 0;
729
+ const frameOptions = { frameVariable: "N", frameOffset, clipStart };
730
+ const opacityKeys = layer.keyframes?.opacity;
731
+ const scaleKeys = layer.keyframes?.scale;
732
+ const opacity = opacityKeys
733
+ ? keyframeChannelExpression("opacity", opacityKeys, frameOptions)
734
+ : formatNumber(layer.opacity);
735
+ const scale = scaleKeys ? keyframeChannelExpression("scale", scaleKeys, frameOptions) : undefined;
736
+ const visible = scale !== undefined ? `if(gt(${scale},0),1,0)` : "1";
737
+ const transition = layer.transition?.role === "in"
738
+ ? transitionProgressExpression({
739
+ frameVariable: "N",
740
+ frameOffset,
741
+ start: layer.transition.start,
742
+ end: layer.transition.end,
743
+ ...(layer.transition.easing !== undefined ? { easing: layer.transition.easing } : {}),
744
+ })
745
+ : undefined;
746
+ const alpha = `min(1,max(0,(${opacity})*(${visible})${transition !== undefined ? `*(${transition})` : ""}))`;
747
+ const width = layer.width ?? stageWidth;
748
+ const height = layer.height ?? stageHeight;
749
+ return `[${inputIndex}:v]format=rgba,geq=r=0:g=0:b=0:a='255*${alpha}',scale=${width}:${height}:flags=neighbor,format=rgba[${outputLabel}]`;
574
750
  }
575
- /** Composites one timeline segment into H.264 or an alpha-preserving FFV1 file. */
576
- export async function compositeTimelineSegment(options) {
751
+ /** W3C soft-light in 0-255 space; FFmpeg's native mode uses a different formula. */
752
+ const SOFT_LIGHT_EXPRESSION = "if(lt(B,128), A-(1-2*B/255)*A*(1-A/255), A+(2*B/255-1)*(if(lt(A,64),((16*A/255-12)*A/255+4)*A/255,sqrt(A/255))*255-A))";
753
+ const BLEND_FILTER_NAMES = {
754
+ multiply: "multiply",
755
+ screen: "screen",
756
+ overlay: "overlay",
757
+ darken: "darken",
758
+ lighten: "lighten",
759
+ "color-dodge": "dodge",
760
+ "color-burn": "burn",
761
+ "hard-light": "hardlight",
762
+ difference: "difference",
763
+ exclusion: "exclusion",
764
+ };
765
+ function blendSpec(mode) {
766
+ if (mode === "soft-light") {
767
+ return {
768
+ options: `c0_expr='${SOFT_LIGHT_EXPRESSION}':c1_expr='${SOFT_LIGHT_EXPRESSION}':c2_expr='${SOFT_LIGHT_EXPRESSION}':c3_mode=normal`,
769
+ sourceFirst: false,
770
+ };
771
+ }
772
+ const name = BLEND_FILTER_NAMES[mode];
773
+ return {
774
+ options: `c0_mode=${name}:c1_mode=${name}:c2_mode=${name}:c3_mode=normal`,
775
+ sourceFirst: mode === "color-dodge" || mode === "color-burn",
776
+ };
777
+ }
778
+ function placementFilter(layer, frameOffset) {
779
+ const clipStart = layer.clipStart ?? 0;
780
+ const frameOptions = { frameVariable: "n", frameOffset, clipStart };
781
+ const xKeys = layer.keyframes?.x;
782
+ const yKeys = layer.keyframes?.y;
783
+ const x = xKeys ? keyframeChannelExpression("x", xKeys, frameOptions) : formatNumber(layer.x);
784
+ const y = yKeys ? keyframeChannelExpression("y", yKeys, frameOptions) : formatNumber(layer.y);
785
+ if (!xKeys && !yKeys) {
786
+ return `x=(main_w-overlay_w)/2+${x}:y=(main_h-overlay_h)/2+${y}`;
787
+ }
788
+ return `x='(main_w-overlay_w)/2+(${x})':y='(main_h-overlay_h)/2+(${y})':eval=frame`;
789
+ }
790
+ /**
791
+ * Builds the FFmpeg filter graph for one timeline composite. Blend layers are
792
+ * placed on a transparent stage-sized canvas, blended with the accumulated
793
+ * backdrop, masked by the source alpha, and composited source-over.
794
+ */
795
+ function buildCompositeGraph(options) {
577
796
  const durationSeconds = options.frames / options.fps;
578
797
  const args = [
579
798
  "-y",
@@ -590,27 +809,126 @@ export async function compositeTimelineSegment(options) {
590
809
  let current = "base0";
591
810
  let inputIndex = 0;
592
811
  let includedIndex = 0;
812
+ const blendedLayers = options.layers.filter((layer) => layer.blend !== "normal" && isLayerVisibleInRange(layer, options.frameOffset, options.frames));
813
+ const canvasLabels = [];
814
+ if (blendedLayers.length > 0) {
815
+ args.push("-f", "lavfi", "-i", `color=c=black@0.0:s=${options.width}x${options.height}:r=${formatNumber(options.fps)}:d=${formatNumber(durationSeconds)}`);
816
+ inputIndex += 1;
817
+ for (let index = 0; index < blendedLayers.length; index += 1) {
818
+ canvasLabels.push(`cv${index}`);
819
+ }
820
+ const canvasChain = `[${inputIndex}:v]format=rgba,colorchannelmixer=aa=0`;
821
+ filters.push(canvasLabels.length === 1
822
+ ? `${canvasChain}[${canvasLabels[0]}]`
823
+ : `${canvasChain},split=${canvasLabels.length}${canvasLabels.map((label) => `[${label}]`).join("")}`);
824
+ }
825
+ let canvasIndex = 0;
593
826
  for (const layer of options.layers) {
594
- if (layer.opacity <= 0 || layer.scale <= 0) {
827
+ if (!isLayerVisibleInRange(layer, options.frameOffset, options.frames)) {
595
828
  continue;
596
829
  }
597
- if (layer.kind === "video") {
830
+ const masked = needsOpacityMask(layer);
831
+ let maskInput = 0;
832
+ if (masked) {
833
+ args.push("-f", "lavfi", "-i",
834
+ // FFmpeg 6 rejects a 1x1 yuv420p source (chroma planes round to zero).
835
+ `color=c=black@0.0:s=2x2:r=${formatNumber(options.fps)}:d=${formatNumber(durationSeconds)}`);
836
+ inputIndex += 1;
837
+ maskInput = inputIndex;
838
+ }
839
+ if (layer.input === "video") {
598
840
  const startSeconds = (layer.sourceStartFrame ?? 0) / options.fps;
599
- args.push("-ss", formatNumber(startSeconds), "-t", formatNumber(durationSeconds), "-i", layer.path, "-an");
841
+ if (options.segment) {
842
+ args.push("-ss", formatNumber(startSeconds), "-t", formatNumber(durationSeconds), "-i", layer.path, "-an");
843
+ }
844
+ else {
845
+ args.push("-ss", formatNumber(startSeconds), "-i", layer.path, "-an");
846
+ }
600
847
  }
601
- else if (layer.kind === "image") {
602
- args.push("-loop", "1", "-framerate", formatNumber(options.fps), "-t", formatNumber(durationSeconds), "-i", layer.path, "-an");
848
+ else if (layer.input === "image") {
849
+ if (options.segment) {
850
+ args.push("-loop", "1", "-framerate", formatNumber(options.fps), "-t", formatNumber(durationSeconds), "-i", layer.path, "-an");
851
+ }
852
+ else {
853
+ args.push("-i", layer.path, "-an");
854
+ }
603
855
  }
604
856
  else {
605
857
  args.push("-i", layer.path);
606
858
  }
607
859
  inputIndex += 1;
608
860
  includedIndex += 1;
609
- filters.push(`[${inputIndex}:v]${layerFilter(layer, options.width, options.height, options.fps)}[l${includedIndex}]`);
610
- filters.push(`[${current}][l${includedIndex}]overlay=x=(main_w-overlay_w)/2+${formatNumber(layer.x)}:y=(main_h-overlay_h)/2+${formatNumber(layer.y)}:format=auto[b${includedIndex}]`);
861
+ const layerInput = inputIndex;
862
+ const base = baseLayerFilters(layer, options.width, options.height, options.fps);
863
+ const presentation = presentationLayerFilters(layer, options.width, options.height, options.frameOffset);
864
+ if (masked) {
865
+ filters.push(`[${layerInput}:v]${base.join(",")}[v${includedIndex}]`);
866
+ filters.push(opacityMaskFilter({
867
+ layer,
868
+ stageWidth: options.width,
869
+ stageHeight: options.height,
870
+ frameOffset: options.frameOffset,
871
+ inputIndex: maskInput,
872
+ outputLabel: `km${includedIndex}`,
873
+ }));
874
+ filters.push(`[v${includedIndex}][km${includedIndex}]blend=c0_mode=normal:c1_mode=normal:c2_mode=normal:c3_mode=multiply${presentation.length > 0 ? `,${presentation.join(",")}` : ""}[l${includedIndex}]`);
875
+ }
876
+ else {
877
+ filters.push(`[${layerInput}:v]${[...base, ...presentation].join(",")}[l${includedIndex}]`);
878
+ }
879
+ if (layer.blend === "normal") {
880
+ filters.push(`[${current}][l${includedIndex}]overlay=${placementFilter(layer, options.frameOffset)}:format=auto[b${includedIndex}]`);
881
+ current = `b${includedIndex}`;
882
+ continue;
883
+ }
884
+ const canvas = canvasLabels[canvasIndex];
885
+ canvasIndex += 1;
886
+ filters.push(`[${canvas}][l${includedIndex}]overlay=${placementFilter(layer, options.frameOffset)}:format=auto,format=rgba,split=2[pb${includedIndex}][pm${includedIndex}]`);
887
+ filters.push(`[${current}]format=rgba,split=3[bc${includedIndex}][bm${includedIndex}][bo${includedIndex}]`);
888
+ filters.push(`[bm${includedIndex}]alphaextract,format=rgba[mask${includedIndex}]`);
889
+ const spec = blendSpec(layer.blend);
890
+ const first = spec.sourceFirst ? `pb${includedIndex}` : `bc${includedIndex}`;
891
+ const second = spec.sourceFirst ? `bc${includedIndex}` : `pb${includedIndex}`;
892
+ filters.push(`[${first}][${second}]blend=${spec.options}[bl${includedIndex}]`);
893
+ filters.push(`[pm${includedIndex}][bl${includedIndex}][mask${includedIndex}]maskedmerge=planes=7[cs${includedIndex}]`);
894
+ filters.push(`[bo${includedIndex}][cs${includedIndex}]overlay=x=0:y=0:format=auto,format=rgba[b${includedIndex}]`);
611
895
  current = `b${includedIndex}`;
612
896
  }
613
- filters.push(`[${current}]format=${options.codec === "ffv1" ? "bgra" : "yuv420p"}[out]`);
897
+ return { args, filters, output: current };
898
+ }
899
+ /** Encode a numbered PNG sequence into a lossless alpha-preserving FFV1 file. */
900
+ export async function encodePngSequenceToFfv1(options) {
901
+ await runCaptured("ffmpeg", [
902
+ "-y",
903
+ "-v",
904
+ "error",
905
+ "-framerate",
906
+ formatNumber(options.fps),
907
+ "-i",
908
+ options.pattern,
909
+ "-c:v",
910
+ "ffv1",
911
+ "-level",
912
+ "3",
913
+ "-pix_fmt",
914
+ "bgra",
915
+ options.outputPath,
916
+ ], { signal: options.signal });
917
+ }
918
+ /** Composites one timeline segment into H.264 or an alpha-preserving FFV1 file. */
919
+ export async function compositeTimelineSegment(options) {
920
+ const graph = buildCompositeGraph({
921
+ width: options.width,
922
+ height: options.height,
923
+ fps: options.fps,
924
+ frames: options.frames,
925
+ frameOffset: options.frameOffset ?? 0,
926
+ background: options.background,
927
+ layers: options.layers,
928
+ segment: true,
929
+ });
930
+ const filters = [...graph.filters, `[${graph.output}]format=${options.codec === "ffv1" ? "bgra" : "yuv420p"}[out]`];
931
+ const args = graph.args;
614
932
  args.push("-filter_complex", filters.join(";"), "-map", "[out]", "-frames:v", String(options.frames), "-r", formatNumber(options.fps), "-an");
615
933
  if (options.codec === "ffv1") {
616
934
  args.push("-c:v", "ffv1", "-level", "3", "-pix_fmt", "bgra", ...containerMuxerArgs("mkv"));
@@ -674,38 +992,18 @@ export async function muxVideoWithAudio(options) {
674
992
  }
675
993
  /** Composites one timeline frame to a PNG without encoding a segment. */
676
994
  export async function compositeTimelineFrame(options) {
677
- const args = [
678
- "-y",
679
- "-v",
680
- "error",
681
- "-f",
682
- "lavfi",
683
- "-i",
684
- `color=c=${ffmpegBackgroundColor(options.background)}:s=${options.width}x${options.height}:r=${formatNumber(options.fps)}:d=${formatNumber(1 / options.fps)}`,
685
- ];
686
- const filters = [
687
- options.background === "transparent" ? "[0:v]format=rgba,colorchannelmixer=aa=0[base0]" : "[0:v]format=rgba[base0]",
688
- ];
689
- let current = "base0";
690
- let inputIndex = 0;
691
- let includedIndex = 0;
692
- for (const layer of options.layers) {
693
- if (layer.opacity <= 0 || layer.scale <= 0) {
694
- continue;
695
- }
696
- if (layer.kind === "video") {
697
- args.push("-ss", formatNumber((layer.sourceStartFrame ?? 0) / options.fps), "-i", layer.path, "-an");
698
- }
699
- else {
700
- args.push("-i", layer.path);
701
- }
702
- inputIndex += 1;
703
- includedIndex += 1;
704
- filters.push(`[${inputIndex}:v]${layerFilter(layer, options.width, options.height, options.fps)}[l${includedIndex}]`);
705
- filters.push(`[${current}][l${includedIndex}]overlay=x=(main_w-overlay_w)/2+${formatNumber(layer.x)}:y=(main_h-overlay_h)/2+${formatNumber(layer.y)}:format=auto[b${includedIndex}]`);
706
- current = `b${includedIndex}`;
707
- }
708
- filters.push(`[${current}]format=rgba[out]`);
995
+ const graph = buildCompositeGraph({
996
+ width: options.width,
997
+ height: options.height,
998
+ fps: options.fps,
999
+ frames: 1,
1000
+ frameOffset: options.frameOffset ?? 0,
1001
+ background: options.background,
1002
+ layers: options.layers,
1003
+ segment: false,
1004
+ });
1005
+ const filters = [...graph.filters, `[${graph.output}]format=rgba[out]`];
1006
+ const args = graph.args;
709
1007
  args.push("-filter_complex", filters.join(";"), "-map", "[out]", "-frames:v", "1", "-update", "1", "-c:v", "png", "-pix_fmt", "rgba", options.outputPath);
710
1008
  await runCaptured("ffmpeg", args, { signal: options.signal });
711
1009
  }