frameyard 0.3.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (168) hide show
  1. package/CHANGELOG.md +117 -0
  2. package/GLOSSARY.md +46 -2
  3. package/README.md +174 -106
  4. package/dist/bindings.d.ts +42 -0
  5. package/dist/bindings.d.ts.map +1 -0
  6. package/dist/bindings.js +144 -0
  7. package/dist/bindings.js.map +1 -0
  8. package/dist/cli.js +32 -21
  9. package/dist/cli.js.map +1 -1
  10. package/dist/composition/adapter.d.ts +30 -0
  11. package/dist/composition/adapter.d.ts.map +1 -0
  12. package/dist/composition/adapter.js +2 -0
  13. package/dist/composition/adapter.js.map +1 -0
  14. package/dist/composition/animation.d.ts +19 -0
  15. package/dist/composition/animation.d.ts.map +1 -0
  16. package/dist/composition/animation.js +55 -0
  17. package/dist/composition/animation.js.map +1 -0
  18. package/dist/composition/attributes.d.ts +7 -0
  19. package/dist/composition/attributes.d.ts.map +1 -0
  20. package/dist/composition/attributes.js +24 -0
  21. package/dist/composition/attributes.js.map +1 -0
  22. package/dist/composition/descriptor.d.ts +30 -0
  23. package/dist/composition/descriptor.d.ts.map +1 -0
  24. package/dist/composition/descriptor.js +188 -0
  25. package/dist/composition/descriptor.js.map +1 -0
  26. package/dist/composition/diagnostics.d.ts +12 -0
  27. package/dist/composition/diagnostics.d.ts.map +1 -0
  28. package/dist/composition/diagnostics.js +38 -0
  29. package/dist/composition/diagnostics.js.map +1 -0
  30. package/dist/composition/index.d.ts +12 -0
  31. package/dist/composition/index.d.ts.map +1 -0
  32. package/dist/composition/index.js +7 -0
  33. package/dist/composition/index.js.map +1 -0
  34. package/dist/composition/media.d.ts +27 -0
  35. package/dist/composition/media.d.ts.map +1 -0
  36. package/dist/composition/media.js +162 -0
  37. package/dist/composition/media.js.map +1 -0
  38. package/dist/composition/paths.d.ts +5 -0
  39. package/dist/composition/paths.d.ts.map +1 -0
  40. package/dist/composition/paths.js +16 -0
  41. package/dist/composition/paths.js.map +1 -0
  42. package/dist/composition/transcripts.d.ts +19 -0
  43. package/dist/composition/transcripts.d.ts.map +1 -0
  44. package/dist/composition/transcripts.js +57 -0
  45. package/dist/composition/transcripts.js.map +1 -0
  46. package/dist/composition/walk.d.ts +10 -0
  47. package/dist/composition/walk.d.ts.map +1 -0
  48. package/dist/composition/walk.js +32 -0
  49. package/dist/composition/walk.js.map +1 -0
  50. package/dist/contracts.d.ts +75 -47
  51. package/dist/contracts.d.ts.map +1 -1
  52. package/dist/contracts.js +14 -159
  53. package/dist/contracts.js.map +1 -1
  54. package/dist/errors.d.ts +15 -4
  55. package/dist/errors.d.ts.map +1 -1
  56. package/dist/errors.js +8 -0
  57. package/dist/errors.js.map +1 -1
  58. package/dist/html/document-adapter.d.ts +5 -0
  59. package/dist/html/document-adapter.d.ts.map +1 -0
  60. package/dist/html/document-adapter.js +17 -0
  61. package/dist/html/document-adapter.js.map +1 -0
  62. package/dist/html/index.d.ts +5 -0
  63. package/dist/html/index.d.ts.map +1 -0
  64. package/dist/html/index.js +3 -0
  65. package/dist/html/index.js.map +1 -0
  66. package/dist/html/parse.d.ts +26 -0
  67. package/dist/html/parse.d.ts.map +1 -0
  68. package/dist/html/parse.js +80 -0
  69. package/dist/html/parse.js.map +1 -0
  70. package/dist/html/validate.d.ts +14 -0
  71. package/dist/html/validate.d.ts.map +1 -0
  72. package/dist/html/validate.js +648 -0
  73. package/dist/html/validate.js.map +1 -0
  74. package/dist/index.d.ts +2 -2
  75. package/dist/index.d.ts.map +1 -1
  76. package/dist/index.js +1 -1
  77. package/dist/index.js.map +1 -1
  78. package/dist/preview/main.d.ts +10 -3
  79. package/dist/preview/main.d.ts.map +1 -1
  80. package/dist/preview/main.js +189 -89
  81. package/dist/preview/main.js.map +1 -1
  82. package/dist/project.d.ts +2 -0
  83. package/dist/project.d.ts.map +1 -1
  84. package/dist/project.js +4 -0
  85. package/dist/project.js.map +1 -1
  86. package/dist/renderer/browser.d.ts +11 -1
  87. package/dist/renderer/browser.d.ts.map +1 -1
  88. package/dist/renderer/browser.js +93 -14
  89. package/dist/renderer/browser.js.map +1 -1
  90. package/dist/renderer/index.d.ts +25 -5
  91. package/dist/renderer/index.d.ts.map +1 -1
  92. package/dist/renderer/index.js +82 -16
  93. package/dist/renderer/index.js.map +1 -1
  94. package/dist/runtime/bindings.d.ts +11 -0
  95. package/dist/runtime/bindings.d.ts.map +1 -0
  96. package/dist/runtime/bindings.js +164 -0
  97. package/dist/runtime/bindings.js.map +1 -0
  98. package/dist/runtime/diagnostics.d.ts +4 -0
  99. package/dist/runtime/diagnostics.d.ts.map +1 -0
  100. package/dist/runtime/diagnostics.js +107 -0
  101. package/dist/runtime/diagnostics.js.map +1 -0
  102. package/dist/runtime/document-adapter.d.ts +4 -0
  103. package/dist/runtime/document-adapter.d.ts.map +1 -0
  104. package/dist/runtime/document-adapter.js +19 -0
  105. package/dist/runtime/document-adapter.js.map +1 -0
  106. package/dist/runtime/host.d.ts +14 -0
  107. package/dist/runtime/host.d.ts.map +1 -0
  108. package/dist/runtime/host.js +596 -0
  109. package/dist/runtime/host.js.map +1 -0
  110. package/dist/runtime/index.d.ts +0 -29
  111. package/dist/runtime/index.d.ts.map +1 -1
  112. package/dist/runtime/index.js +0 -360
  113. package/dist/runtime/index.js.map +1 -1
  114. package/dist/runtime/media.d.ts.map +1 -1
  115. package/dist/runtime/media.js +20 -51
  116. package/dist/runtime/media.js.map +1 -1
  117. package/dist/scaffold.d.ts.map +1 -1
  118. package/dist/scaffold.js +177 -84
  119. package/dist/scaffold.js.map +1 -1
  120. package/dist/server/composition-html.d.ts +20 -0
  121. package/dist/server/composition-html.d.ts.map +1 -0
  122. package/dist/server/composition-html.js +45 -0
  123. package/dist/server/composition-html.js.map +1 -0
  124. package/dist/server/preview-html.d.ts.map +1 -1
  125. package/dist/server/preview-html.js +19 -14
  126. package/dist/server/preview-html.js.map +1 -1
  127. package/dist/server/server.d.ts.map +1 -1
  128. package/dist/server/server.js +94 -6
  129. package/dist/server/server.js.map +1 -1
  130. package/dist/version.d.ts +1 -1
  131. package/dist/version.js +1 -1
  132. package/docs/adr/0001-keep-composition-authoring-web-native.md +2 -0
  133. package/docs/adr/0006-seek-embedded-media-from-the-frame-clock.md +7 -1
  134. package/docs/adr/0007-make-html-documents-the-composition-unit.md +31 -0
  135. package/docs/adr/0008-share-the-composition-document-grammar.md +18 -0
  136. package/docs/adr/0009-composite-production-frames-in-software.md +37 -0
  137. package/docs/migration-0.4.md +279 -0
  138. package/docs/v0.1-design.md +78 -56
  139. package/examples/demo/README.md +8 -4
  140. package/examples/demo/compositions/demo.controller.js +120 -0
  141. package/examples/demo/compositions/demo.html +253 -0
  142. package/examples/demo/video.config.js +1 -1
  143. package/examples/video/README.md +61 -0
  144. package/examples/video/assets/clip.mp4 +0 -0
  145. package/examples/video/assets/clip.webm +0 -0
  146. package/examples/video/compositions/video-mp4.html +362 -0
  147. package/examples/video/compositions/video.html +362 -0
  148. package/examples/video/props/video.json +8 -0
  149. package/examples/video/scripts/make-clip.sh +109 -0
  150. package/examples/video/video.config.js +6 -0
  151. package/examples/webgl/README.md +30 -0
  152. package/examples/webgl/compositions/webgl.controller.js +200 -0
  153. package/examples/webgl/compositions/webgl.html +215 -0
  154. package/examples/webgl/props/webgl.json +6 -0
  155. package/examples/webgl/video.config.js +5 -0
  156. package/package.json +7 -3
  157. package/skills/frameyard/README.md +4 -0
  158. package/skills/frameyard/SKILL.md +40 -12
  159. package/skills/frameyard/references/audio-assets.md +25 -14
  160. package/skills/frameyard/references/bindings.md +73 -0
  161. package/skills/frameyard/references/common-mistakes.md +59 -0
  162. package/skills/frameyard/references/controllers.md +126 -0
  163. package/skills/frameyard/references/deterministic-authoring.md +143 -52
  164. package/skills/frameyard/references/diagnostics.md +90 -0
  165. package/skills/frameyard/references/failure-recovery.md +37 -14
  166. package/skills/frameyard/references/json-results.md +43 -9
  167. package/skills/frameyard/references/subtitles-karaoke.md +10 -18
  168. package/examples/demo/compositions/demo.js +0 -206
package/README.md CHANGED
@@ -1,8 +1,8 @@
1
1
  # Frameyard
2
2
 
3
- Frameyard is a local-first framework for building short, parameterized videos with HTML, CSS, and JavaScript and rendering them deterministically.
3
+ Frameyard is a local-first framework for building short, parameterized videos with HTML, CSS, and an optional JavaScript controller, and rendering them deterministically.
4
4
 
5
- The v0.1 design is implemented: composition contracts, browser runtime and preview, `init`, `dev`, `check`, `still`, `render`, `tts`, `transcribe`, `doctor`, deterministic Chromium capture, embedded video and WebGL, FFmpeg output, provenance manifests, transcript validation, and the official agent skill.
5
+ The 0.4 design is implemented: HTML composition documents, declarative bindings, CSS animation scheduling, frame-state variables, inline and external controllers, the composition host, static validation with structured diagnostics, iframe preview, direct render/check/still sessions, audio and transcripts, deterministic Chromium capture, FFmpeg output, and the official agent skill.
6
6
 
7
7
  ## Requirements
8
8
 
@@ -28,7 +28,7 @@ npx frameyard dev
28
28
 
29
29
  `dev` prints a local preview URL. Select a composition, edit its JSON props, scrub frames, and play the composition with its audio timeline.
30
30
 
31
- The scaffold includes an `AGENTS.md` guide for coding agents working in the project, covering the determinism rules and the `check`, `still`, and `render` workflow. It also includes a `package.json` with `dev`, `check`, `render`, and `still` scripts pinned to the scaffolding CLI version, so `pnpm dev` or `npm run dev` works without a global install.
31
+ The scaffold includes an `AGENTS.md` guide for coding agents working in the project, covering the HTML authoring rules and the `check`, `still`, and `render` workflow. It also includes a `package.json` with `dev`, `check`, `render`, and `still` scripts pinned to the scaffolding CLI version, so `pnpm dev` or `npm run dev` works without a global install, and a `.gitignore` that ignores `out/`.
32
32
 
33
33
  `npx frameyard` resolves the latest published version. Pin it with `npx frameyard@<version>`, or install the package locally, when a project needs a fixed CLI version.
34
34
 
@@ -37,142 +37,195 @@ Validate and render:
37
37
  ```sh
38
38
  npx frameyard doctor
39
39
  npx frameyard check hello
40
- npx frameyard render hello --props props/hello.json --out hello.mp4
40
+ npx frameyard render hello --props props/hello.json
41
41
  npx frameyard still hello --frame 42 --out hello-frame.png
42
42
  ```
43
43
 
44
- The 30-second reference composition lives in [`examples/demo`](./examples/demo). Its README has the commands for running the demo from a repository checkout.
44
+ `render` writes the video to `out/hello.mp4` in the project by default; pass `--out <file.mp4>` to choose another path. The scaffold's `.gitignore` ignores `out/`.
45
+
46
+ The 30-second reference composition lives in [`examples/demo`](./examples/demo), a focused WebGL shader example lives in [`examples/webgl`](./examples/webgl), and a two-pass embedded-video example with WebM and MP4 sources lives in [`examples/video`](./examples/video). Each README has the commands for running it from a repository checkout.
45
47
 
46
48
  ## Project format
47
49
 
48
- A project is ordinary HTML, CSS, assets, and native JavaScript ES modules. `video.config.js` explicitly lists composition entry points; there is no directory scanning:
50
+ A project is ordinary HTML, CSS, assets, and optional native JavaScript controller modules. `video.config.js` explicitly lists composition entry points; there is no directory scanning:
49
51
 
50
52
  ```js
51
53
  export default {
52
54
  compositions: {
53
- hello: "./compositions/hello.js",
55
+ hello: "./compositions/hello.html",
54
56
  },
55
57
  };
56
58
  ```
57
59
 
58
- A composition default-exports a definition. `defineComposition` is optional but gives early metadata validation and editor types:
60
+ A composition is a full HTML document with one `<fy-composition>` root. Metadata, default props, markup, CSS, bindings, audio, transcripts, and an optional controller all live in the document:
59
61
 
60
- ```js
61
- import { defineComposition } from "frameyard";
62
- import { random } from "frameyard/runtime";
63
-
64
- let title;
65
-
66
- export default defineComposition({
67
- width: 1920,
68
- height: 1080,
69
- fps: 30,
70
- durationInFrames: 900,
71
- defaultProps: { title: "Hello" },
72
- validateProps(props) {
73
- return typeof props.title === "string" ? [] : ["title must be a string"];
74
- },
75
- audio: [{ src: "assets/hit.wav", startFrame: 15, volume: 0.8 }],
62
+ ```html
63
+ <!doctype html>
64
+ <html>
65
+ <head>
66
+ <meta charset="utf-8">
67
+ <script type="application/json" data-fy="props">
68
+ { "title": "Hello", "accent": "#b8f36b" }
69
+ </script>
70
+ <style>
71
+ @keyframes fade-up {
72
+ from { opacity: 0; transform: translateY(24px); }
73
+ to { opacity: 1; transform: translateY(0); }
74
+ }
75
+ #title { animation: fade-up 1s ease-out both; }
76
+ </style>
77
+ </head>
78
+ <body>
79
+ <fy-composition width="1920" height="1080" fps="30" duration-in-frames="900">
80
+ <h1 id="title"
81
+ data-fy-text="title"
82
+ data-fy-anim-start="0"
83
+ data-fy-anim-duration="30"></h1>
84
+ <div id="progress" style="width: calc(var(--fy-progress) * 100%)"></div>
85
+ </fy-composition>
86
+ </body>
87
+ </html>
88
+ ```
76
89
 
77
- mount({ container }) {
78
- container.innerHTML = "<h1 id='title'></h1>";
79
- title = container.querySelector("#title");
80
- },
90
+ `width`, `height`, `fps`, and `duration-in-frames` are fixed. Props are JSON-serializable, deep-frozen, and merged shallowly over the `data-fy="props"` defaults. They cannot change metadata.
81
91
 
82
- update({ frame, props }) {
83
- title.textContent = props.title;
84
- title.style.transform = `translateX(${random("title:" + frame) * 40}px)`;
85
- },
92
+ ### Bindings
86
93
 
87
- dispose() {
88
- title = undefined;
89
- },
90
- });
94
+ - `data-fy-text="path"` sets text content.
95
+ - `data-fy-attr-src="path"` (and other `data-fy-attr-*` names) sets an attribute.
96
+ - `data-fy-class-active="path"` toggles a class.
97
+ - `data-fy-if="path"` removes the element when the value is false.
98
+ - `<template data-fy-repeat="items">` clones its content for each item, with `item` and `index` in scope and `--fy-index` on each clone.
99
+ - Paths are dot-separated (`user.name`, `items.0.label`). Missing paths warn and resolve to an empty value.
100
+ - Primitive top-level props are also exposed as `--fy-prop-<name>` CSS custom properties.
101
+
102
+ Bindings apply once when a composition loads, before the controller mounts. Controller-created DOM is the controller's responsibility.
103
+
104
+ ### Animation
105
+
106
+ CSS `@keyframes` and the Web Animations API are the animation language. The runtime pauses every animation and seeks it from the frame clock.
107
+
108
+ `data-fy-anim-start`, `data-fy-anim-duration`, and `data-fy-anim-easing` schedule one animation per element in composition frames. The CSS `animation` shorthand supplies keyframes and fill mode; the attributes override its delay and duration.
109
+
110
+ The runtime sets frame-state variables on `:root`:
111
+
112
+ | Variable | Meaning |
113
+ | --- | --- |
114
+ | `--fy-frame` | Current frame |
115
+ | `--fy-time` | Current time in seconds |
116
+ | `--fy-progress` | Progress from `0` to `1` |
117
+ | `--fy-fps` | Composition frame rate |
118
+ | `--fy-duration` | Duration in seconds |
119
+ | `--fy-duration-frames` | Duration in frames |
120
+
121
+ Use `--fy-progress` for progress bars and `--fy-frame` or `--fy-time` in `calc()` for simple computed motion. CSS transitions are not deterministic and are diagnosed.
122
+
123
+ ### Controller
124
+
125
+ An optional controller handles what HTML and CSS cannot express. It is declared exactly once, either inline or as an external module:
126
+
127
+ ```html
128
+ <script type="text/fy-controller">
129
+ export default {
130
+ mount({ root, props }) { /* ... */ },
131
+ update({ frame, props, root }) { /* ... */ },
132
+ dispose() { /* ... */ },
133
+ validateProps(props) { /* ... */ },
134
+ };
135
+ </script>
136
+ ```
137
+
138
+ ```html
139
+ <script type="text/fy-controller" src="./hello.controller.js"></script>
91
140
  ```
92
141
 
93
- The runtime supplies an import map for `frameyard` and `frameyard/runtime`, so project code needs no bundling step and no `node_modules` import.
142
+ - `mount()` may be asynchronous; `update()` and `validateProps()` are synchronous and seek-safe; `dispose()` releases resources.
143
+ - Inline controllers are the default for small, single-use controllers. Extract to an external module for Canvas 2D, WebGL, karaoke, reuse, tests, or a large controller.
144
+ - A validation-only controller may export just `validateProps(props)`.
145
+ - The controller is the only executable code in a composition. Other executable scripts and `on*` handlers are rejected.
146
+ - Inline source is served through a virtual module URL, so relative imports and `import.meta.url` behave like an external module. A literal `</script>` inside inline source must be escaped.
94
147
 
95
148
  ### Composition lifecycle
96
149
 
97
- - `mount({ container, props })` creates the visual structure once and may finish asynchronous work.
98
- - `update({ frame, props })` makes that structure represent the frame. It must be synchronous and seek-safe.
99
- - `dispose()` releases resources after the last frame.
150
+ The composition host loads the document, applies bindings, imports the controller, calls `mount()`, prepares media, and seeks to frame 0. `check`, `still`, and `render` then seek explicitly to requested frames and wait for embedded media and browser paint before evaluating or capturing. `dispose()` runs before the host releases the document.
100
151
 
101
- The only animation inputs are the integer frame and immutable props. Metadata (`width`, `height`, `fps`, `durationInFrames`) is fixed and validated before mounting. Props are JSON-serializable, deep-frozen, and merged shallowly over `defaultProps`.
152
+ The controller lifecycle is `mount → update → dispose`. `update({ frame, props })` must be synchronous and seek-safe. Use `random("stable:key")` from `frameyard/runtime` instead of `Math.random()`. Independent `requestAnimationFrame` loops, timers, wall-clock reads, and CSS transitions are not deterministic animation drivers.
102
153
 
103
- Use `random("stable:key")` instead of `Math.random()`. The runtime pauses CSS animations and the Web Animations API inside the composition container and seeks them to the current frame. Independent `requestAnimationFrame` loops, timers, transitions, and wall-clock reads are not deterministic animation drivers.
154
+ `check` runs static composition validation before launching Chromium. It validates metadata, the root and controller declarations, default props, bindings, animation attributes, scripts and handlers, unsupported visual elements, and asset references, then reports stable diagnostic codes with source locations.
104
155
 
105
156
  ## Embedded video and WebGL
106
157
 
107
- Embedded videos are frame-seeked by the runtime. Place a `<video>` element in `mount()` and describe its place on the composition timeline with data attributes:
158
+ Embedded videos are declared in the composition document and frame-seeked by the runtime:
108
159
 
109
160
  ```html
110
161
  <video
111
162
  src="assets/clip.webm"
112
- data-frameyard-start="30"
113
- data-frameyard-trim="0.5"
114
- data-frameyard-duration="4"
115
- data-frameyard-volume="0.8"
163
+ data-fy-start="30"
164
+ data-fy-trim="0.5"
165
+ data-fy-duration="4"
166
+ data-fy-volume="0.8"
116
167
  ></video>
117
168
  ```
118
169
 
119
170
  | Attribute | Meaning | Default |
120
171
  | --- | --- | --- |
121
- | `data-frameyard-start` | Composition frame where the source starts | `0` |
122
- | `data-frameyard-trim` | Seconds trimmed from the source start | `0` |
123
- | `data-frameyard-duration` | Optional seconds of source to use | rest of source |
124
- | `data-frameyard-volume` | Embedded audio volume from `0` to `1` | `1` |
172
+ | `data-fy-start` | Composition frame where the source starts | `0` |
173
+ | `data-fy-trim` | Seconds trimmed from the source start | `0` |
174
+ | `data-fy-duration` | Optional seconds of source to use | rest of source |
175
+ | `data-fy-volume` | Embedded audio volume from `0` to `1` | `1` |
176
+
177
+ The runtime pauses every video and seeks it to the frame-derived source time. Do not call `play()`, `pause()`, or set `currentTime` in a controller. Videos must be in the composition document's light DOM; shadow roots are not scanned. Video sources are local project-relative files; live streams, remote URLs, and sources that cannot be seeked are not supported. The runtime loads the complete file during the loading phase, so frame production performs no media network requests.
125
178
 
126
- The runtime pauses every video and seeks it to the frame-derived source time. Do not call `play()`, `pause()`, or set `currentTime` in `update()`. Create video elements and set these attributes in `mount()`: a video added or changed during `update()` cannot be prepared without network access and is rejected. Videos must be in the composition container's light DOM; shadow roots are not scanned. Video sources are project-relative local files; live streams, remote URLs, and sources that cannot be seeked are not supported. The runtime loads the complete file during the loading phase, so frame production performs no media network requests.
179
+ An unmuted `<video>` also contributes its soundtrack to the same timeline as declarative audio clips. Frameyard extracts that track with FFmpeg for production and mirrors it in preview, while keeping the video element itself silent. Set `muted` or `data-fy-audio="off"` to use a video as a visual-only asset. See [`examples/video`](./examples/video) for the same composition driven by a VP9/Opus WebM clip and an H.264/AAC MP4 clip.
127
180
 
128
- An unmuted `<video>` also contributes its soundtrack to the same timeline as declarative `audio` clips. Frameyard extracts that track with FFmpeg for production and mirrors it in preview, while keeping the video element itself silent. Set `muted` or `data-frameyard-audio="off"` to use a video as a visual-only asset. The attributes above also control where its audio enters the mix.
181
+ Fade the overlay layers, not a full-scene ancestor of a video. Chromium can reuse the load frame in interactive preview after a large seek when an ancestor animates opacity. Production captures keep the composition hidden until the first seek, so stills resolve to the requested frame.
129
182
 
130
- WebGL 1 and WebGL 2 canvases are captured through the normal frame path. Draw the complete frame from `update()` and do not depend on the previous frame's drawing buffer; clear or redraw the scene each time. Production sessions pin software WebGL (SwiftShader) so rendered pixels do not vary with the host GPU.
183
+ WebGL 1 and WebGL 2 canvases are captured through the normal frame path. Draw the complete frame from the controller's `update()` and do not depend on the previous frame's drawing buffer; clear or redraw the scene each time. Production sessions pin software WebGL (SwiftShader) so rendered pixels do not vary with the host GPU. See [`examples/webgl`](./examples/webgl) for a complete shader example.
131
184
 
132
185
  ## Audio timeline
133
186
 
134
- Audio clips are declarative:
187
+ Audio clips are `<audio>` elements:
135
188
 
136
- ```js
137
- audio: [
138
- {
139
- src: "assets/voice.wav", // project-relative local file
140
- startFrame: 30, // composition timeline position
141
- trimStart: 0.25, // seconds trimmed from the source
142
- trimDuration: 4.2, // optional seconds to use
143
- volume: 0.8, // 0..1
144
- },
145
- ];
189
+ ```html
190
+ <audio src="assets/voice.wav"
191
+ data-fy-start="30"
192
+ data-fy-trim="0.25"
193
+ data-fy-duration="4.2"
194
+ data-fy-volume="0.8"></audio>
146
195
  ```
147
196
 
148
- Preview synchronizes the timeline in the browser. Production mixes the same clips offline with FFmpeg. Unmuted embedded videos contribute their audio tracks to this mix unless they opt out. `check` reports clip durations and warnings when a clip extends beyond the composition. Frameyard never time-stretches audio or changes composition duration to fit it.
197
+ - Placement is in composition frames.
198
+ - Trimming is in seconds from the source.
199
+ - Audio, video, and transcript `src` values resolve from the project root.
200
+ - Frameyard never time-stretches audio or changes composition duration to fit it.
201
+ - `check` reports clip durations and overrun diagnostics.
202
+
203
+ Preview synchronizes the timeline in the browser. Production mixes the same clips offline with FFmpeg. Unmuted embedded videos contribute their audio tracks to this mix unless they opt out.
149
204
 
150
205
  ## Transcripts
151
206
 
152
- A composition can declare canonical transcript assets that `check`, `still`, and `render` must validate:
207
+ A composition declares canonical transcript assets with `<fy-transcript>`:
153
208
 
154
- ```js
155
- transcripts: [
156
- { src: "assets/voice.transcript.json", words: true },
157
- ],
209
+ ```html
210
+ <fy-transcript src="assets/voice.transcript.json" words></fy-transcript>
158
211
  ```
159
212
 
160
- Transcript paths are project-relative. `words: true` requires actual measured word timings for karaoke-style presentation; Frameyard fails with `TRANSCRIPT_WORDS_UNAVAILABLE` rather than interpolating them. See [subtitles and karaoke](./skills/frameyard/references/subtitles-karaoke.md) for the canonical JSON shape and the seek-safe frame conversion recipe.
213
+ Transcript paths resolve from the project root. `words` requires actual measured word timings for karaoke-style presentation; Frameyard fails with `TRANSCRIPT_WORDS_UNAVAILABLE` rather than interpolating them. `check`, `still`, and `render` validate declared transcripts. Visual presentation is the composition's responsibility; a controller can read the transcript JSON and render timed text or karaoke. See [subtitles and karaoke](./skills/frameyard/references/subtitles-karaoke.md) for the canonical JSON shape and the seek-safe frame conversion recipe.
161
214
 
162
215
  ## Commands
163
216
 
164
217
  | Command | Purpose |
165
218
  | --- | --- |
166
- | `frameyard init [directory]` | Scaffold a minimal project, including an `AGENTS.md` agent guide and pinned `package.json` scripts |
219
+ | `frameyard init [directory]` | Scaffold a minimal HTML project, including an `AGENTS.md` agent guide and pinned `package.json` scripts |
167
220
  | `frameyard dev [--port <port>]` | Start the preview server |
168
- | `frameyard check <id> [--props <file>]` | Validate props, assets, audio, transcripts, and every frame without capture |
221
+ | `frameyard check <id> [--props <file>]` | Run static validation, then evaluate every frame without capture |
169
222
  | `frameyard still <id> --frame <n>` | Capture one PNG through the production path |
170
- | `frameyard render <id> --out <video.mp4>` | Capture frames with Chromium and encode with FFmpeg, optionally in parallel |
223
+ | `frameyard render <id> [--out <video.mp4>]` | Capture frames with Chromium and encode with FFmpeg, optionally in parallel; defaults to `out/<id>.mp4` in the project |
171
224
  | `frameyard tts ...` | Produce one frozen 24 kHz mono WAV with local qwentts.cpp |
172
225
  | `frameyard transcribe ...` | Produce canonical transcript JSON with optional word timings and SRT/VTT exports |
173
226
  | `frameyard doctor` | Inspect core requirements, optional integrations, and manifest integrity |
174
227
 
175
- Common options: `--project <dir>`, `--json`, `--help`, and `--version`. `still` and `render` accept `--overwrite`; `render` also accepts `--concurrency <n>` (1-16); `tts` and `transcribe` accept `--force`. `init` takes the target directory as a positional argument. Output paths are relative to the current working directory.
228
+ Common options: `--project <dir>`, `--json`, `--help`, and `--version`. `still` and `render` accept `--overwrite`; `render` also accepts `--concurrency <n>` (1-16); `tts` and `transcribe` accept `--force`. `init` takes the target directory as a positional argument. Explicit output paths passed with `--out` are relative to the current working directory; `render` defaults to `out/<id>.mp4` inside the project root.
176
229
 
177
230
  A render fails immediately on an uncaught error, unhandled rejection, missing required asset, or invalid composition output. It writes to a temporary file and publishes only after FFmpeg succeeds. Existing outputs are not replaced unless `--overwrite` is supplied.
178
231
 
@@ -183,27 +236,33 @@ Non-interactive commands support `--json`. Results go to stdout and human progre
183
236
  ```json
184
237
  {
185
238
  "ok": true,
186
- "command": "render",
239
+ "command": "check",
187
240
  "composition": "hello",
188
- "path": "/absolute/path/hello.mp4",
189
- "width": 1920,
190
- "height": 1080,
191
- "fps": 30,
192
- "durationInFrames": 900,
193
- "framesRendered": 900,
194
- "concurrency": 1,
195
- "audio": { "clips": 1, "warnings": [] },
196
- "transcripts": { "clips": 1, "warnings": [] }
241
+ "entry": "compositions/hello.html",
242
+ "controller": { "kind": "inline" },
243
+ "metadata": { "width": 1920, "height": 1080, "fps": 30, "durationInFrames": 900 },
244
+ "framesChecked": 900,
245
+ "audio": { "clips": 1 },
246
+ "transcripts": { "clips": 0 },
247
+ "diagnostics": []
197
248
  }
198
249
  ```
199
250
 
200
- Errors use the same envelope:
251
+ Diagnostics carry `severity`, `code`, `message`, and optional `details` with source locations, element or attribute names, values, and suggestions. Errors use the same envelope and may include `error.details` and top-level `diagnostics`:
201
252
 
202
253
  ```json
203
254
  {
204
255
  "ok": false,
205
- "command": "render",
206
- "error": { "code": "OUTPUT_EXISTS", "message": "Output already exists: ..." }
256
+ "command": "check",
257
+ "error": { "code": "BINDING_INVALID", "message": "Unknown Frameyard attribute ..." },
258
+ "diagnostics": [
259
+ {
260
+ "severity": "error",
261
+ "code": "BINDING_INVALID",
262
+ "message": "Unknown Frameyard attribute \"data-fy-txt\".",
263
+ "details": { "path": "compositions/hello.html", "line": 12, "column": 5, "attribute": "data-fy-txt" }
264
+ }
265
+ ]
207
266
  }
208
267
  ```
209
268
 
@@ -254,54 +313,63 @@ npx frameyard doctor --talker-model models/talker.gguf --codec-model models/code
254
313
 
255
314
  ## Agent skill
256
315
 
257
- The optional host-neutral agent skill lives in [`skills/frameyard/`](./skills/frameyard). It uses the same project format and CLI as humans, documents deterministic authoring, audio assets, subtitle/karaoke recipes, JSON results, and failure recovery, and ships a thin CLI wrapper. The npm package includes the skill files matching its API version. Install it with `npx skills add vladiantio/frameyard`, or see [`skills/frameyard/README.md`](./skills/frameyard/README.md) for host-specific installation or referencing.
316
+ The optional host-neutral agent skill lives in [`skills/frameyard/`](./skills/frameyard). It uses the same project format and CLI as humans, documents HTML authoring, bindings, controllers, static diagnostics, audio assets, subtitle/karaoke recipes, JSON results, and failure recovery, and ships a thin CLI wrapper. The npm package includes the skill files matching its API version. Install it with `npx skills add vladiantio/frameyard`, or see [`skills/frameyard/README.md`](./skills/frameyard/README.md) for host-specific installation or referencing.
258
317
 
259
318
  ## Rendering and reproducibility
260
319
 
261
320
  Production rendering:
262
321
 
263
- 1. Starts the same preview server and browser runtime used by `dev`.
264
- 2. Launches one pinned Chromium session, or several independent sessions when `--concurrency` is greater than 1.
265
- 3. Seeks explicitly to each frame and waits for composition updates.
266
- 4. Captures the viewport through the Chrome DevTools Protocol.
267
- 5. Pipes PNG frames in timeline order and the offline audio mix to FFmpeg.
268
- 6. Encodes H.264 video with `yuv420p` and AAC audio into an MP4.
322
+ 1. Runs static composition validation and resolves metadata before launching Chromium.
323
+ 2. Navigates directly to the composition document with the composition host.
324
+ 3. Launches one pinned Chromium session, or several independent sessions when `--concurrency` is greater than 1.
325
+ 4. Seeks explicitly to each frame, calls the controller, seeks CSS/WAAPI animations, and waits for embedded media decode and browser paint.
326
+ 5. Captures the viewport through the Chrome DevTools Protocol.
327
+ 6. Pipes PNG frames in timeline order and the offline audio mix to FFmpeg.
328
+ 7. Encodes H.264 video with `yuv420p` and AAC audio into an MP4.
269
329
 
270
330
  Rendering is sequential by default. Pass `--concurrency <n>` (1-16) to seek and capture frames with up to `n` independent browser sessions; the effective value is capped by the composition's frame count. Captured frames are always encoded in timeline order, so parallel and sequential renders of the same inputs produce equivalent visual frames. Concurrency only changes scheduling: it does not change composition behavior or the determinism contract. Parallel rendering uses more CPU and memory, so choose a value appropriate for the machine.
271
331
 
272
332
  For the same composition revision, props, assets, frame, and pinned runtime/browser environment, the visual state is identical. Encoded bytes are not a reproducibility guarantee, even on the same machine; they depend on the FFmpeg build, encoder threading, and runtime environment.
273
333
 
274
- For a repeatable local measurement of seek, capture, encode, and full-render time, run `pnpm bench`. The demo and the controlled fixtures under `bench/fixtures/` are the reference workloads.
334
+ For a repeatable local measurement of seek, capture, encode, and full-render time, run `pnpm bench`. See `bench/README.md` for the measurement recipe and browser-flag experiments. The demo and the controlled fixtures under `bench/fixtures/` are the reference workloads.
275
335
 
276
336
  ## Current limits
277
337
 
278
338
  Deferred from this implementation slice:
279
339
 
280
- - Built-in subtitle rendering, automatic caption burn-in, and karaoke rendering
340
+ - Built-in subtitle rendering, automatic caption burn-in, and declarative karaoke elements
341
+ - Nested `data-fy-repeat` scopes and a general template language
342
+ - Declarative JSON Schema validation
343
+ - Arbitrary executable scripts in composition HTML
281
344
  - `qwentts.cpp` reference-audio voice cloning and VoiceDesign
282
345
  - Automatic long-text chunking, streaming synthesis, and alternative generated-audio formats
283
346
  - Automatic upstream builds, model downloads, environment repair, and hosted rendering
284
- - Frame caches and general image-sequence output
347
+ - Frame caches, batch still capture, and general image-sequence output
285
348
  - React or other UI-framework adapters, plugin APIs, and agent-only runtimes
286
349
  - TypeScript transpilation for project compositions and npm dependency optimization
287
350
 
288
- See the [v0.1 design baseline](./docs/v0.1-design.md) for the full product contract and the [domain glossary](./GLOSSARY.md) for terminology.
351
+ See the [design baseline](./docs/v0.1-design.md) for the full product contract, [ADR 0007](./docs/adr/0007-make-html-documents-the-composition-unit.md) for the HTML authoring decision, and the [domain glossary](./GLOSSARY.md) for terminology.
289
352
 
290
353
  ## Repository layout
291
354
 
292
- - `src/contracts.ts`: shared composition, props, audio, and transcript requirement contracts
355
+ - `src/contracts.ts`: shared composition, props, audio, transcript, and diagnostic contracts
356
+ - `src/bindings.ts`: browser-safe binding path and value semantics
357
+ - `src/composition/`: browser-safe composition document reader, descriptor, per-kind declarations, and adapter contract
358
+ - `src/html/`: Node-side parse5 adapter and static composition validation
293
359
  - `src/transcript.ts`: canonical transcript validation and SRT/VTT formatting
294
- - `src/runtime/`: browser lifecycle, frame clock, deterministic randomness
295
- - `src/preview/`: interactive preview client
296
- - `src/server/`: native ES-module server and import map
297
- - `src/renderer/`: Chromium capture, audio mixing, transcript inspection, FFmpeg encoding
360
+ - `src/runtime/`: browser host, frame clock, deterministic randomness, bindings, media, diagnostics
361
+ - `src/preview/`: interactive iframe preview client
362
+ - `src/server/`: native ES-module server, host injection, and inline-controller route
363
+ - `src/renderer/`: Chromium capture, static validation integration, audio mixing, transcript inspection, FFmpeg encoding
298
364
  - `src/assets/`: local TTS/transcription wrappers, executable resolution, atomic publishing, provenance
299
365
  - `src/doctor.ts`: environment, integration, and manifest inspection
300
366
  - `src/scaffold.ts`: `init` scaffold templates, including the generated package manifest, project README, and agent guide
301
367
  - `src/cli.ts`: command-line entry point
302
368
  - `skills/`: official host-neutral agent skill
303
369
  - `examples/demo/`: the 30-second reference composition
304
- - `bench/fixtures/`: controlled compositions for the render benchmark
370
+ - `examples/webgl/`: the focused WebGL shader example
371
+ - `examples/video/`: the two-pass embedded-video example with WebM and MP4 sources
372
+ - `bench/fixtures/`: controlled HTML compositions for the render benchmark
305
373
  - `scripts/`: benchmark, release checks, versioning, and git-hook installation
306
374
  - `docs/adr/`: architecture decision records
307
375
 
@@ -0,0 +1,42 @@
1
+ import type { Props } from "./contracts.js";
2
+ export interface RepeatScope {
3
+ item: unknown;
4
+ index: number;
5
+ }
6
+ export interface BindingScope {
7
+ props: Readonly<Props>;
8
+ repeat?: RepeatScope;
9
+ }
10
+ export interface BindingPathValidation {
11
+ ok: boolean;
12
+ reason?: string;
13
+ }
14
+ export interface ResolvedBindingPath {
15
+ found: boolean;
16
+ value: unknown;
17
+ }
18
+ export interface CoercedBindingValue<T> {
19
+ value: T;
20
+ invalid?: boolean;
21
+ }
22
+ /** Validates a dot-separated binding path without resolving it. */
23
+ export declare function validateBindingPath(path: string): BindingPathValidation;
24
+ /**
25
+ * Resolves a dot-separated path against props, or against the repeat scope when
26
+ * the first segment is `item` or `index`. Only own properties are read.
27
+ */
28
+ export declare function resolveBindingPath(scope: BindingScope, path: string): ResolvedBindingPath;
29
+ /** JavaScript truthiness, used by `data-fy-if` and class bindings. */
30
+ export declare function isTruthy(value: unknown): boolean;
31
+ /** Coerces a value for `textContent`. Objects and arrays are invalid. */
32
+ export declare function coerceText(value: unknown): CoercedBindingValue<string>;
33
+ /**
34
+ * Coerces a value for an attribute. `true` writes an empty attribute, and
35
+ * `false`, `null`, and `undefined` remove it.
36
+ */
37
+ export declare function coerceAttribute(value: unknown): CoercedBindingValue<string | null>;
38
+ /** Coerces a value for a class binding. Objects and arrays are invalid. */
39
+ export declare function coerceClass(value: unknown): CoercedBindingValue<boolean>;
40
+ /** Suggests the closest top-level prop name for a mistyped binding path. */
41
+ export declare function suggestBindingPath(path: string, props: Readonly<Props>): string | undefined;
42
+ //# sourceMappingURL=bindings.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"bindings.d.ts","sourceRoot":"","sources":["../src/bindings.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAE5C,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;IACvB,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,MAAM,WAAW,qBAAqB;IACpC,EAAE,EAAE,OAAO,CAAC;IACZ,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,mBAAmB;IAClC,KAAK,EAAE,OAAO,CAAC;IACf,KAAK,EAAE,OAAO,CAAC;CAChB;AAED,MAAM,WAAW,mBAAmB,CAAC,CAAC;IACpC,KAAK,EAAE,CAAC,CAAC;IACT,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAKD,mEAAmE;AACnE,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,qBAAqB,CAqBvE;AAED;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,YAAY,EAAE,IAAI,EAAE,MAAM,GAAG,mBAAmB,CA6BzF;AAED,sEAAsE;AACtE,wBAAgB,QAAQ,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAEhD;AAED,yEAAyE;AACzE,wBAAgB,UAAU,CAAC,KAAK,EAAE,OAAO,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAWtE;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,OAAO,GAAG,mBAAmB,CAAC,MAAM,GAAG,IAAI,CAAC,CAWlF;AAED,2EAA2E;AAC3E,wBAAgB,WAAW,CAAC,KAAK,EAAE,OAAO,GAAG,mBAAmB,CAAC,OAAO,CAAC,CAWxE;AAED,4EAA4E;AAC5E,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,GAAG,MAAM,GAAG,SAAS,CAqB3F"}
@@ -0,0 +1,144 @@
1
+ const PATH_SEGMENT = /^[A-Za-z0-9_$-]+$/;
2
+ const DANGEROUS_SEGMENTS = new Set(["__proto__", "constructor", "prototype"]);
3
+ /** Validates a dot-separated binding path without resolving it. */
4
+ export function validateBindingPath(path) {
5
+ if (path.length === 0) {
6
+ return { ok: false, reason: "Binding path must not be empty." };
7
+ }
8
+ for (const segment of path.split(".")) {
9
+ if (segment.length === 0) {
10
+ return { ok: false, reason: "Binding path must not contain empty segments." };
11
+ }
12
+ if (DANGEROUS_SEGMENTS.has(segment)) {
13
+ return { ok: false, reason: `Binding path must not use "${segment}".` };
14
+ }
15
+ if (!PATH_SEGMENT.test(segment)) {
16
+ return {
17
+ ok: false,
18
+ reason: `Binding path segment "${segment}" must use letters, digits, "_", "$", or "-".`,
19
+ };
20
+ }
21
+ }
22
+ return { ok: true };
23
+ }
24
+ /**
25
+ * Resolves a dot-separated path against props, or against the repeat scope when
26
+ * the first segment is `item` or `index`. Only own properties are read.
27
+ */
28
+ export function resolveBindingPath(scope, path) {
29
+ if (!validateBindingPath(path).ok) {
30
+ return { found: false, value: undefined };
31
+ }
32
+ const segments = path.split(".");
33
+ const first = segments[0];
34
+ let current;
35
+ let index = 0;
36
+ if (scope.repeat && (first === "item" || first === "index")) {
37
+ current = first === "item" ? scope.repeat.item : scope.repeat.index;
38
+ index = 1;
39
+ }
40
+ else {
41
+ current = scope.props;
42
+ }
43
+ for (; index < segments.length; index += 1) {
44
+ const segment = segments[index];
45
+ if (current === null || typeof current !== "object") {
46
+ return { found: false, value: undefined };
47
+ }
48
+ if (!Object.prototype.hasOwnProperty.call(current, segment)) {
49
+ return { found: false, value: undefined };
50
+ }
51
+ current = current[segment];
52
+ }
53
+ return { found: true, value: current };
54
+ }
55
+ /** JavaScript truthiness, used by `data-fy-if` and class bindings. */
56
+ export function isTruthy(value) {
57
+ return Boolean(value);
58
+ }
59
+ /** Coerces a value for `textContent`. Objects and arrays are invalid. */
60
+ export function coerceText(value) {
61
+ if (value === null || value === undefined) {
62
+ return { value: "" };
63
+ }
64
+ if (typeof value === "string") {
65
+ return { value };
66
+ }
67
+ if (typeof value === "number" || typeof value === "boolean") {
68
+ return { value: String(value) };
69
+ }
70
+ return { value: "", invalid: true };
71
+ }
72
+ /**
73
+ * Coerces a value for an attribute. `true` writes an empty attribute, and
74
+ * `false`, `null`, and `undefined` remove it.
75
+ */
76
+ export function coerceAttribute(value) {
77
+ if (value === true) {
78
+ return { value: "" };
79
+ }
80
+ if (value === false || value === null || value === undefined) {
81
+ return { value: null };
82
+ }
83
+ if (typeof value === "string" || typeof value === "number") {
84
+ return { value: String(value) };
85
+ }
86
+ return { value: null, invalid: true };
87
+ }
88
+ /** Coerces a value for a class binding. Objects and arrays are invalid. */
89
+ export function coerceClass(value) {
90
+ if (value === null || value === undefined) {
91
+ return { value: false };
92
+ }
93
+ if (typeof value === "boolean") {
94
+ return { value };
95
+ }
96
+ if (typeof value === "string" || typeof value === "number") {
97
+ return { value: Boolean(value) };
98
+ }
99
+ return { value: false, invalid: true };
100
+ }
101
+ /** Suggests the closest top-level prop name for a mistyped binding path. */
102
+ export function suggestBindingPath(path, props) {
103
+ const root = path.split(".")[0];
104
+ if (root === undefined || root.length === 0) {
105
+ return undefined;
106
+ }
107
+ let best;
108
+ let bestDistance = Number.POSITIVE_INFINITY;
109
+ for (const key of Object.keys(props)) {
110
+ if (key === root) {
111
+ return undefined;
112
+ }
113
+ const distance = editDistance(root.toLowerCase(), key.toLowerCase());
114
+ if (distance < bestDistance) {
115
+ bestDistance = distance;
116
+ best = key;
117
+ }
118
+ }
119
+ return bestDistance <= 2 ? best : undefined;
120
+ }
121
+ function editDistance(left, right) {
122
+ if (left === right) {
123
+ return 0;
124
+ }
125
+ if (left.length === 0) {
126
+ return right.length;
127
+ }
128
+ if (right.length === 0) {
129
+ return left.length;
130
+ }
131
+ let previous = Array.from({ length: right.length + 1 }, (_, index) => index);
132
+ for (let leftIndex = 1; leftIndex <= left.length; leftIndex += 1) {
133
+ const current = [leftIndex];
134
+ for (let rightIndex = 1; rightIndex <= right.length; rightIndex += 1) {
135
+ const substitution = previous[rightIndex - 1] + (left[leftIndex - 1] === right[rightIndex - 1] ? 0 : 1);
136
+ const insertion = current[rightIndex - 1] + 1;
137
+ const deletion = previous[rightIndex] + 1;
138
+ current[rightIndex] = Math.min(substitution, insertion, deletion);
139
+ }
140
+ previous = current;
141
+ }
142
+ return previous[right.length];
143
+ }
144
+ //# sourceMappingURL=bindings.js.map