@tongue-core/engine 0.1.0-rc.1 → 0.1.0-rc.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +17 -0
  3. package/dist/assets/codecs/types.d.ts +2 -0
  4. package/dist/assets/codecs/wav.d.ts +3 -0
  5. package/dist/assets/codecs/wav.js +26 -6
  6. package/dist/assets/store.d.ts +18 -0
  7. package/dist/assets/store.js +82 -6
  8. package/dist/audio/mix.d.ts +7 -1
  9. package/dist/audio/mix.js +14 -3
  10. package/dist/evaluate/evaluate.js +123 -10
  11. package/dist/evaluate/tree.d.ts +24 -1
  12. package/dist/index.d.ts +5 -4
  13. package/dist/index.js +5 -4
  14. package/dist/layout/layout.js +130 -82
  15. package/dist/paint/geometry.d.ts +12 -4
  16. package/dist/paint/geometry.js +25 -11
  17. package/dist/paint/list.d.ts +28 -1
  18. package/dist/paint/list.js +26 -2
  19. package/dist/paint/morph.js +11 -7
  20. package/dist/paint/paint.d.ts +18 -7
  21. package/dist/paint/paint.js +181 -16
  22. package/dist/paint/stroke.d.ts +26 -0
  23. package/dist/paint/stroke.js +233 -0
  24. package/dist/raster/raster.d.ts +3 -0
  25. package/dist/raster/raster.js +5 -0
  26. package/dist/raster/wasm-bytes.js +2 -2
  27. package/dist/render/compose.d.ts +1 -1
  28. package/dist/render/compose.js +33 -11
  29. package/dist/render/renderer.d.ts +7 -6
  30. package/dist/render/renderer.js +28 -7
  31. package/dist/render/size.d.ts +6 -0
  32. package/dist/render/size.js +7 -0
  33. package/dist/render/sounds.d.ts +4 -1
  34. package/dist/render/sounds.js +6 -2
  35. package/dist/style/computed.d.ts +14 -1
  36. package/dist/style/parse.js +2 -4
  37. package/dist/style/resolve.js +53 -4
  38. package/dist/text/fonts.d.ts +4 -1
  39. package/dist/text/fonts.js +9 -1
  40. package/dist/text/ttf.js +5 -1
  41. package/package.json +6 -4
  42. package/raster.wasm +0 -0
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Backpine
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md CHANGED
@@ -17,6 +17,23 @@ renderer.render(120, { scale: 0.5 });
17
17
  const rgba = renderer.rgba8(); // or yuv420(), png()
18
18
  ```
19
19
 
20
+ **In a Cloudflare Worker**, which may not compile WebAssembly from bytes, import the raster as a
21
+ module (`@tongue-core/engine/raster.wasm`, the same bytes as the embedded copy) and pass it in:
22
+
23
+ ```ts
24
+ import { FilmRenderer, Raster } from "@tongue-core/engine";
25
+ import raster from "@tongue-core/engine/raster.wasm"; // a WebAssembly.Module in a Worker
26
+
27
+ const renderer = await FilmRenderer.create(plan, assets, await Raster.create(raster));
28
+ ```
29
+
30
+ With little memory (a Worker), hold only what each frame draws: `displayList` throws `ImageNotLoaded`
31
+ naming the image a frame needs, and `renderer.keepOnly(list)` frees the rest from the raster before
32
+ `renderer.draw(list)`. `apps/frames-worker/src/runner.ts` does this.
33
+
34
+ TypeScript needs `declare module "*.wasm" { const module: WebAssembly.Module; export default module; }`
35
+ somewhere in the Worker's sources.
36
+
20
37
  - **Frames are independent.** Nothing carries over between frames except caches, so frames
21
38
  render in any order, on any number of machines.
22
39
  - **Video frames come from the host** (`FrameOptions.video`, a `VideoFrameSource`): ffmpeg on a
@@ -6,6 +6,8 @@ export interface RgbaImage {
6
6
  }
7
7
  /** Planar float audio in -1..1. Every channel has the same length. */
8
8
  export interface PcmAudio {
9
+ /** Source sample represented by channels[0][0], for a window decoded from segmented audio. */
10
+ readonly offset?: number;
9
11
  readonly sampleRate: number;
10
12
  readonly channels: ReadonlyArray<Float32Array>;
11
13
  }
@@ -1,3 +1,6 @@
1
1
  import type { PcmAudio } from "./types.js";
2
2
  export declare function readWav(bytes: Uint8Array): PcmAudio;
3
+ /** 32-bit float WAV: every sample kept exactly as it is, with no dither and no clipping. For audio
4
+ * between steps (a stem on its way to a processor), where 16 bits would round it each time. */
5
+ export declare function writeFloatWav(audio: PcmAudio): Uint8Array;
3
6
  export declare function writeWav(audio: PcmAudio, seed?: number): Uint8Array;
@@ -96,7 +96,9 @@ function random(seed) {
96
96
  return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
97
97
  };
98
98
  }
99
- export function writeWav(audio, seed = 0x5eed) {
99
+ /** A WAV header for `frames` of `channels` samples of `bits` each, in the given format; the
100
+ * samples go after it, at byte 44. */
101
+ function header(audio, format, bits) {
100
102
  const channels = audio.channels.length;
101
103
  const frames = audio.channels[0]?.length ?? 0;
102
104
  if (channels === 0)
@@ -105,21 +107,39 @@ export function writeWav(audio, seed = 0x5eed) {
105
107
  fail("channels have different lengths");
106
108
  if (!Number.isInteger(audio.sampleRate) || audio.sampleRate <= 0)
107
109
  fail(`bad sample rate ${audio.sampleRate}`);
108
- const dataLength = frames * channels * 2;
110
+ const bytes = bits / 8;
111
+ const dataLength = frames * channels * bytes;
109
112
  const out = new Uint8Array(44 + dataLength);
110
113
  const view = new DataView(out.buffer);
111
114
  out.set([0x52, 0x49, 0x46, 0x46]); // RIFF
112
115
  view.setUint32(4, 36 + dataLength, true);
113
116
  out.set([0x57, 0x41, 0x56, 0x45, 0x66, 0x6d, 0x74, 0x20], 8); // WAVEfmt␠
114
117
  view.setUint32(16, 16, true);
115
- view.setUint16(20, FORMAT_PCM, true);
118
+ view.setUint16(20, format, true);
116
119
  view.setUint16(22, channels, true);
117
120
  view.setUint32(24, audio.sampleRate, true);
118
- view.setUint32(28, audio.sampleRate * channels * 2, true);
119
- view.setUint16(32, channels * 2, true);
120
- view.setUint16(34, 16, true);
121
+ view.setUint32(28, audio.sampleRate * channels * bytes, true);
122
+ view.setUint16(32, channels * bytes, true);
123
+ view.setUint16(34, bits, true);
121
124
  out.set([0x64, 0x61, 0x74, 0x61], 36); // data
122
125
  view.setUint32(40, dataLength, true);
126
+ return { out, view, frames };
127
+ }
128
+ /** 32-bit float WAV: every sample kept exactly as it is, with no dither and no clipping. For audio
129
+ * between steps (a stem on its way to a processor), where 16 bits would round it each time. */
130
+ export function writeFloatWav(audio) {
131
+ const { out, view, frames } = header(audio, FORMAT_FLOAT, 32);
132
+ let at = 44;
133
+ for (let f = 0; f < frames; f++) {
134
+ for (const channel of audio.channels) {
135
+ view.setFloat32(at, channel[f] ?? 0, true);
136
+ at += 4;
137
+ }
138
+ }
139
+ return out;
140
+ }
141
+ export function writeWav(audio, seed = 0x5eed) {
142
+ const { view, out, frames } = header(audio, FORMAT_PCM, 16);
123
143
  // TPDF dither: the difference of two uniform variables spans ±1 LSB with a triangular
124
144
  // distribution, which decorrelates the rounding error from the signal.
125
145
  const next = random(seed);
@@ -21,6 +21,7 @@ export interface LoadedVideo {
21
21
  readonly poster: LoadedImage | null;
22
22
  }
23
23
  export interface LoadedAssets {
24
+ readonly audioWindow?: (name: string, fromSeconds: number, toSeconds: number) => PcmAudio | null;
24
25
  readonly fonts: FontLibrary;
25
26
  readonly images: ReadonlyMap<string, LoadedImage>;
26
27
  readonly audio: ReadonlyMap<string, PcmAudio>;
@@ -65,6 +66,9 @@ export declare function loadAssets(assets: ReadonlyArray<Asset> | undefined, fet
65
66
  * need): the maps a renderer reads at draw time, filled as files come. Drawing a frame whose
66
67
  * files aren't here yet throws, as with any missing asset; nothing is drawn half loaded. */
67
68
  export declare class AssetShelf implements LoadedAssets {
69
+ private readonly pendingPosters;
70
+ private readonly audioInfo;
71
+ private readonly audioParts;
68
72
  readonly fonts: FontLibrary;
69
73
  readonly images: Map<string, LoadedImage>;
70
74
  readonly audio: Map<string, PcmAudio>;
@@ -75,6 +79,20 @@ export declare class AssetShelf implements LoadedAssets {
75
79
  add(asset: Asset, path: string, bytes: Uint8Array, audio?: boolean): void;
76
80
  /** Whether the file at `path` is on the shelf. */
77
81
  has(path: string): boolean;
82
+ remove(path: string): void;
83
+ readonly audioWindow: (name: string, fromSeconds: number, toSeconds: number) => PcmAudio | null;
84
+ }
85
+ export declare function parseAudioInfo(bytes: Uint8Array): {
86
+ readonly sampleRate: number;
87
+ readonly segments: ReadonlyArray<{
88
+ readonly first: number;
89
+ readonly samples: number;
90
+ }>;
91
+ };
92
+ /** A frame needs an image that isn't here: a host loading on demand fetches `asset` and draws again. */
93
+ export declare class ImageNotLoaded extends AssetError {
94
+ readonly asset: ImageAsset;
95
+ constructor(asset: ImageAsset);
78
96
  }
79
97
  export declare function imageOf(assets: LoadedAssets, asset: ImageAsset): LoadedImage;
80
98
  export declare function videoOf(assets: LoadedAssets, asset: VideoAsset): LoadedVideo;
@@ -82,6 +82,9 @@ export async function loadAssets(assets = declaredAssets(), fetch, options = {})
82
82
  * need): the maps a renderer reads at draw time, filled as files come. Drawing a frame whose
83
83
  * files aren't here yet throws, as with any missing asset; nothing is drawn half loaded. */
84
84
  export class AssetShelf {
85
+ pendingPosters = new Map();
86
+ audioInfo = new Map();
87
+ audioParts = new Map();
85
88
  fonts = new FontLibrary();
86
89
  images = new Map();
87
90
  audio = new Map();
@@ -97,18 +100,33 @@ export class AssetShelf {
97
100
  case "image":
98
101
  this.images.set(asset.name, { qoi: bytes, ...qoiSize(bytes) });
99
102
  break;
100
- case "audio":
101
- if (audio)
103
+ case "audio": {
104
+ if (!audio)
105
+ break;
106
+ if (path.endsWith("/info.json"))
107
+ this.audioInfo.set(asset.name, parseAudioInfo(bytes));
108
+ else if (path.includes(`/audio/${asset.name}/`) || path.startsWith(`audio/${asset.name}/`)) {
109
+ const index = Number(/\/(\d+)\.wav$/.exec(path)?.[1]);
110
+ if (!Number.isInteger(index))
111
+ throw new AssetError(`Invalid audio segment ${path}.`);
112
+ const parts = this.audioParts.get(asset.name) ?? new Map();
113
+ parts.set(index, readWav(bytes));
114
+ this.audioParts.set(asset.name, parts);
115
+ }
116
+ else
102
117
  this.audio.set(asset.name, readWav(bytes));
103
118
  break;
119
+ }
104
120
  case "video": {
105
121
  const known = this.videos.get(asset.name);
106
- if (path === videoInfoPath(asset))
107
- this.videos.set(asset.name, { info: parseVideoInfo(bytes, asset.name), poster: known?.poster ?? null });
122
+ if (path === videoInfoPath(asset)) {
123
+ this.videos.set(asset.name, { info: parseVideoInfo(bytes, asset.name), poster: known?.poster ?? this.pendingPosters.get(asset.name) ?? null });
124
+ this.pendingPosters.delete(asset.name);
125
+ }
108
126
  else if (path === videoPosterPath(asset) && known)
109
127
  this.videos.set(asset.name, { info: known.info, poster: { qoi: bytes, ...qoiSize(bytes) } });
110
128
  else if (path === videoPosterPath(asset))
111
- throw new AssetError(`video "${asset.name}": its poster came before its info.json.`);
129
+ this.pendingPosters.set(asset.name, { qoi: bytes, ...qoiSize(bytes) });
112
130
  break;
113
131
  }
114
132
  }
@@ -118,11 +136,69 @@ export class AssetShelf {
118
136
  has(path) {
119
137
  return this.held.has(path);
120
138
  }
139
+ remove(path) {
140
+ this.held.delete(path);
141
+ const image = /^images\/([^/]+)\.qoi$/.exec(path)?.[1];
142
+ if (image !== undefined)
143
+ this.images.delete(image);
144
+ const poster = /^video\/([^/]+)\/poster\.qoi$/.exec(path)?.[1];
145
+ const video = poster === undefined ? undefined : this.videos.get(poster);
146
+ if (poster !== undefined)
147
+ this.pendingPosters.delete(poster);
148
+ if (poster !== undefined && video)
149
+ this.videos.set(poster, { ...video, poster: null });
150
+ const audio = /^audio\/([^/]+)\/(\d+)\.wav$/.exec(path);
151
+ if (audio?.[1] !== undefined)
152
+ this.audioParts.get(audio[1])?.delete(Number(audio[2]));
153
+ }
154
+ audioWindow = (name, fromSeconds, toSeconds) => {
155
+ const info = this.audioInfo.get(name);
156
+ if (!info)
157
+ return this.audio.get(name) ?? null;
158
+ const first = Math.max(0, Math.floor(fromSeconds * info.sampleRate));
159
+ const end = Math.ceil(toSeconds * info.sampleRate);
160
+ const channels = [new Float32Array(Math.max(0, end - first)), new Float32Array(Math.max(0, end - first))];
161
+ for (let k = 0; k < info.segments.length; k++) {
162
+ const segment = info.segments[k];
163
+ if (!segment || segment.first >= end || segment.first + segment.samples <= first)
164
+ continue;
165
+ const part = this.audioParts.get(name)?.get(k);
166
+ if (!part)
167
+ throw new AssetError(`audio "${name}": segment ${k} is not loaded.`);
168
+ if (part.sampleRate !== info.sampleRate || part.channels.length !== 2 || part.channels.some((channel) => channel.length !== segment.samples))
169
+ throw new AssetError(`audio "${name}": segment ${k} has a different rate, channel count or length from its metadata.`);
170
+ const a = Math.max(first, segment.first), b = Math.min(end, segment.first + segment.samples);
171
+ channels.forEach((channel, c) => channel.set(part.channels[c]?.subarray(a - segment.first, b - segment.first) ?? new Float32Array(b - a), a - first));
172
+ }
173
+ return { sampleRate: info.sampleRate, channels, offset: first };
174
+ };
175
+ }
176
+ export function parseAudioInfo(bytes) {
177
+ const value = JSON.parse(new TextDecoder().decode(bytes));
178
+ if (!isRecord(value) || value["sampleRate"] !== 48000 || value["channels"] !== 2 || !Array.isArray(value["segments"]))
179
+ throw new AssetError("Invalid segmented audio metadata.");
180
+ let end = 0;
181
+ const segments = value["segments"].map((entry) => {
182
+ if (!isRecord(entry) || entry["first"] !== end || typeof entry["samples"] !== "number" || !Number.isInteger(entry["samples"]) || entry["samples"] <= 0)
183
+ throw new AssetError("Audio segments must be contiguous and have a positive sample count.");
184
+ const segment = { first: end, samples: entry["samples"] };
185
+ end += segment.samples;
186
+ return segment;
187
+ });
188
+ return { sampleRate: 48000, segments };
189
+ }
190
+ /** A frame needs an image that isn't here: a host loading on demand fetches `asset` and draws again. */
191
+ export class ImageNotLoaded extends AssetError {
192
+ asset;
193
+ constructor(asset) {
194
+ super(`Image "${asset.name}" is not loaded (was it declared with image() and imported?).`);
195
+ this.asset = asset;
196
+ }
121
197
  }
122
198
  export function imageOf(assets, asset) {
123
199
  const image = assets.images.get(asset.name);
124
200
  if (!image)
125
- throw new AssetError(`Image "${asset.name}" is not loaded (was it declared with image() and imported?).`);
201
+ throw new ImageNotLoaded(asset);
126
202
  return image;
127
203
  }
128
204
  export function videoOf(assets, asset) {
@@ -43,4 +43,10 @@ export interface WindowInput {
43
43
  /** The part of the soundtrack between two frames: what a player mixes a few seconds at a time, so
44
44
  * an hour-long film never needs its whole soundtrack in memory. Exactly that part of `mix`, except
45
45
  * where a clip changes speed (time-stretching is padded, not continued from the window before). */
46
- export declare function mixWindow({ fps, clips, sounds, from, to }: WindowInput): PcmAudio;
46
+ export declare function mixWindow(input: WindowInput): PcmAudio;
47
+ /** The peak limiter the soundtrack ends with, on audio summed some other way (stems mixed
48
+ * separately, then added together). Per sample, so `limitAudio(busWindow(x))` is `mixWindow(x)`. */
49
+ export declare function limitAudio(audio: PcmAudio): PcmAudio;
50
+ /** What `mixWindow` sums before it limits: the clips and sounds between two frames, stereo at
51
+ * 48 kHz, unlimited. A stem (some of a film's clips, mixed on their own) is this over those clips. */
52
+ export declare function busWindow({ fps, clips, sounds, from, to }: WindowInput): PcmAudio;
package/dist/audio/mix.js CHANGED
@@ -94,9 +94,10 @@ function channelAt(pcm, channel, fromSeconds, length) {
94
94
  return out;
95
95
  const ratio = pcm.sampleRate / SAMPLE_RATE;
96
96
  const offset = fromSeconds * pcm.sampleRate;
97
+ const windowOffset = pcm.offset ?? 0;
97
98
  for (let i = 0; i < length; i++) {
98
99
  const pos = offset + i * ratio;
99
- const j = Math.floor(pos), f = pos - j;
100
+ const index = Math.floor(pos), j = index - windowOffset, f = pos - index;
100
101
  out[i] = (src[j] ?? 0) * (1 - f) + (src[j + 1] ?? 0) * f;
101
102
  }
102
103
  return out;
@@ -216,7 +217,17 @@ const STRETCH_PAD = 4 * WINDOW;
216
217
  /** The part of the soundtrack between two frames: what a player mixes a few seconds at a time, so
217
218
  * an hour-long film never needs its whole soundtrack in memory. Exactly that part of `mix`, except
218
219
  * where a clip changes speed (time-stretching is padded, not continued from the window before). */
219
- export function mixWindow({ fps, clips, sounds, from, to }) {
220
+ export function mixWindow(input) {
221
+ return limitAudio(busWindow(input));
222
+ }
223
+ /** The peak limiter the soundtrack ends with, on audio summed some other way (stems mixed
224
+ * separately, then added together). Per sample, so `limitAudio(busWindow(x))` is `mixWindow(x)`. */
225
+ export function limitAudio(audio) {
226
+ return { sampleRate: audio.sampleRate, channels: audio.channels.map(limit) };
227
+ }
228
+ /** What `mixWindow` sums before it limits: the clips and sounds between two frames, stereo at
229
+ * 48 kHz, unlimited. A stem (some of a film's clips, mixed on their own) is this over those clips. */
230
+ export function busWindow({ fps, clips, sounds, from, to }) {
220
231
  const start = Math.round((from / fps) * SAMPLE_RATE);
221
232
  const length = Math.max(0, Math.ceil((to / fps) * SAMPLE_RATE) - start);
222
233
  const left = new Float32Array(length), right = new Float32Array(length);
@@ -242,5 +253,5 @@ export function mixWindow({ fps, clips, sounds, from, to }) {
242
253
  add(left, sound, at, gain);
243
254
  add(right, sound, at, gain);
244
255
  }
245
- return { sampleRate: SAMPLE_RATE, channels: [limit(left), limit(right)] };
256
+ return { sampleRate: SAMPLE_RATE, channels: [left, right] };
246
257
  }
@@ -1,4 +1,4 @@
1
- import { composePose, easeOf, enterPose, enterSpec, exitLength, exitPose, exitSpec, flatten, frameOf, isAt, isImageAsset, isVideoAsset, keysOf, rest, timeAt, withTime, } from "@tongue-core/film/model";
1
+ import { composePose, easeOf, enterPose, enterSpec, exitLength, exitPose, exitSpec, flatten, frameOf, isAt, isImageAsset, isVideoAsset, keysOf, node, rest, timeAt, withTime, } from "@tongue-core/film/model";
2
2
  import { schedule, typedAt } from "./typing.js";
3
3
  import { parseColor, StyleError } from "../style/parse.js";
4
4
  import { computeStyle, defaults, overrideOf, recordOf, rootText, textStyleOf } from "../style/resolve.js";
@@ -25,6 +25,7 @@ export function evaluate(content, frame, env) {
25
25
  class Evaluator {
26
26
  frame;
27
27
  env;
28
+ parent;
28
29
  camera = null;
29
30
  pointer = null;
30
31
  sounds = [];
@@ -33,14 +34,23 @@ class Evaluator {
33
34
  keyStack = [];
34
35
  /** Where each narration's word matching has got to (Says match words in order). */
35
36
  sayCursors = new Map();
36
- constructor(frame, env) {
37
+ /** Set for one of MotionBlur's samples: it is a moment between frames, not the frame. */
38
+ sampling;
39
+ /** `parent`: the evaluator a MotionBlur sample passes its sounds to; `null` drops them. */
40
+ constructor(frame, env, parent = undefined) {
37
41
  this.frame = frame;
38
42
  this.env = env;
43
+ this.parent = parent;
44
+ this.sampling = parent !== undefined;
39
45
  }
40
46
  controls() {
41
47
  return { camera: this.camera, pointer: this.pointer, sounds: this.sounds, switches: this.switches };
42
48
  }
43
49
  sound(kind, at) {
50
+ if (this.sampling) {
51
+ this.parent?.sound(kind, at);
52
+ return;
53
+ }
44
54
  if (at >= this.frame && at < this.frame + this.env.soundWindow)
45
55
  this.sounds.push({ kind, at });
46
56
  }
@@ -78,6 +88,8 @@ class Evaluator {
78
88
  case "element": return this.element(c.tag, c.props, c.children, ctx);
79
89
  case "show": return this.show(c, ctx);
80
90
  case "camera": {
91
+ if (this.sampling)
92
+ throw new EvalError("<Camera> can't go inside <MotionBlur>: put the MotionBlur inside the camera's world.");
81
93
  this.camera = { node: c, shots: nodesOf(c.children).flatMap((s) => (s.kind === "shot" ? [s] : [])) };
82
94
  return [];
83
95
  }
@@ -91,6 +103,8 @@ class Evaluator {
91
103
  return [{ kind: "box", box: { ...this.stageBox(c.style, kids, ctx.text), pose: ctx.pose } }];
92
104
  }
93
105
  case "center": return this.center(c.style, c.children, ctx);
106
+ case "mask": return this.mask(c, ctx);
107
+ case "motion-blur": return this.motionBlur(c, ctx);
94
108
  case "shot":
95
109
  case "move":
96
110
  case "click": return [];
@@ -104,7 +118,8 @@ class Evaluator {
104
118
  element(tag, props, children, ctx) {
105
119
  const styleRecord = recordOf(props["style"], `<${tag}> style`);
106
120
  const id = typeof props["id"] === "string" ? props["id"] : null;
107
- const morph = typeof props["morph"] === "string" ? props["morph"] : null;
121
+ // Every sample would hold the same key, and a key means one element. Inside a blur there is none.
122
+ const morph = !this.sampling && typeof props["morph"] === "string" ? props["morph"] : null;
108
123
  this.noteKey(morph);
109
124
  const style = computeStyle(styleRecord, ctx.text, tag);
110
125
  if (style.display === "none")
@@ -129,7 +144,7 @@ class Evaluator {
129
144
  }
130
145
  const hovered = overrideOf(props["hovered"], `<${tag}> hovered`);
131
146
  const pressed = overrideOf(props["pressed"], `<${tag}> pressed`);
132
- const shell = { tag, id, morph, style, hovered, pressed, pose: ctx.pose, screen: false, grow: ctx.grow };
147
+ const shell = { tag, id, morph, style, hovered, pressed, pose: ctx.pose, screen: false, grow: ctx.grow, corners: cornersOf(props["corners"], style, tag) };
133
148
  if (tag === "img") {
134
149
  const src = props["src"];
135
150
  if (!isImageAsset(src))
@@ -145,10 +160,7 @@ class Evaluator {
145
160
  const paths = flatten(children).flatMap((k) => {
146
161
  if (typeof k === "string" || k.kind !== "element" || k.tag !== "path")
147
162
  return [];
148
- const d = k.props["d"], fill = k.props["fill"], rule = k.props["fillRule"];
149
- if (typeof d !== "string")
150
- throw new StyleError("<path> needs d.");
151
- return [{ verbs: parsePath(d), fill: typeof fill === "string" ? parseColor(fill) : style.text.color, rule: rule === "evenodd" ? "evenodd" : "nonzero" }];
163
+ return [pathOf(k.props, style.text)];
152
164
  });
153
165
  return [{ kind: "box", box: { ...shell, content: { kind: "svg", view: parseViewBox(view), paths } } }];
154
166
  }
@@ -158,6 +170,8 @@ class Evaluator {
158
170
  /** The frame a <video> shows now: playback starts at `start` (scene frames) and runs through
159
171
  * the file from `in` to `out` at `speed`, optionally reversed, looped or frozen. */
160
172
  videoPicture(props) {
173
+ if (this.sampling)
174
+ throw new EvalError("<video> can't go inside <MotionBlur>: footage has its own blur. Put it beside the MotionBlur.");
161
175
  const src = props["src"];
162
176
  if (!isVideoAsset(src))
163
177
  throw new EvalError("<video> needs src={a video asset}.");
@@ -253,6 +267,8 @@ class Evaluator {
253
267
  return out;
254
268
  }
255
269
  pointerNode(p) {
270
+ if (this.sampling)
271
+ throw new EvalError("<Pointer> can't go inside <MotionBlur>: it belongs to the scene.");
256
272
  const steps = nodesOf(p.steps);
257
273
  const moves = steps.flatMap((s) => (s.kind === "move" ? [s] : []));
258
274
  const clicks = steps.flatMap((s) => (s.kind === "click" ? [s] : []));
@@ -340,10 +356,52 @@ class Evaluator {
340
356
  const from = f.from === null ? -Infinity : frameOf(f.from);
341
357
  if (this.frame < from)
342
358
  return [];
343
- this.noteKey(f.morph);
359
+ const morph = this.sampling ? null : f.morph;
360
+ this.noteKey(morph);
344
361
  const pose = composePose(ctx.pose, enterPose(enterSpec(f.enter), this.frame - from, this.env.fps));
345
362
  const style = computeStyle({ position: "absolute", inset: 0, background: f.color }, ctx.text, "div");
346
- return [{ kind: "box", box: { tag: "fill", id: null, morph: f.morph, hovered: null, pressed: null, pose, screen: true, grow: 1, style, content: { kind: "boxes", children: [] } } }];
363
+ return [{ kind: "box", box: { tag: "fill", id: null, morph, hovered: null, pressed: null, pose, screen: true, grow: 1, style, content: { kind: "boxes", children: [] } } }];
364
+ }
365
+ mask(m, ctx) {
366
+ const style = computeStyle({ position: "absolute", inset: 0, ...m.style }, ctx.text, "div");
367
+ const inner = { text: style.text, pose: rest, grow: 1 };
368
+ const part = (content, tag) => ({
369
+ tag, id: null, morph: null, hovered: null, pressed: null, pose: rest, screen: false, grow: 1,
370
+ style: computeStyle({ position: "absolute", inset: 0, display: "stack" }, style.text, "div"),
371
+ content: { kind: "boxes", children: this.asBoxes(this.children(content, inner), style.text) },
372
+ });
373
+ const box = {
374
+ tag: "mask", id: null, morph: null, hovered: null, pressed: null, pose: ctx.pose, screen: false, grow: ctx.grow, style,
375
+ content: { kind: "boxes", children: [part(m.children, "mask-content"), part([m.by], "mask-shape")] },
376
+ group: { kind: "mask", invert: m.invert },
377
+ };
378
+ return [{ kind: "box", box }];
379
+ }
380
+ /** The same content at several moments around this frame, centered on it. Each sample has its own
381
+ * evaluator at its own time. Only the middle one's sounds count: the rest are the same sounds. */
382
+ motionBlur(n, ctx) {
383
+ if (!(n.shutter >= 0 && n.shutter <= 720))
384
+ throw new EvalError(`<MotionBlur> shutter: expected 0 to 720 degrees, got ${n.shutter}.`);
385
+ if (!Number.isInteger(n.samples) || n.samples < 2 || n.samples > 64)
386
+ throw new EvalError(`<MotionBlur> samples: expected a whole number from 2 to 64, got ${n.samples}.`);
387
+ const style = computeStyle({ position: "absolute", inset: 0, ...n.style }, ctx.text, "div");
388
+ const inner = { text: style.text, pose: rest, grow: 1 };
389
+ const open = n.shutter / 360;
390
+ const children = Array.from({ length: n.samples }, (_, i) => {
391
+ const at = this.frame + (i / (n.samples - 1) - 0.5) * open;
392
+ const ev = new Evaluator(at, this.env, i === n.samples >> 1 ? this : null);
393
+ const items = withTime(timeAt(at, this.env.fps), () => ev.children([node({ kind: "component", name: "MotionBlur", render: n.render })], inner));
394
+ return {
395
+ tag: "motion-sample", id: null, morph: null, hovered: null, pressed: null, pose: rest, screen: false, grow: 1,
396
+ style: computeStyle({ position: "absolute", inset: 0, display: "stack" }, style.text, "div"),
397
+ content: { kind: "boxes", children: ev.asBoxes(items, style.text) },
398
+ };
399
+ });
400
+ const box = {
401
+ tag: "motion-blur", id: null, morph: null, hovered: null, pressed: null, pose: ctx.pose, screen: false, grow: ctx.grow, style,
402
+ content: { kind: "boxes", children }, group: { kind: "samples" },
403
+ };
404
+ return [{ kind: "box", box }];
347
405
  }
348
406
  center(styleRecord, children, ctx) {
349
407
  const style = computeStyle(styleRecord, ctx.text, "div");
@@ -361,6 +419,61 @@ class Evaluator {
361
419
  }
362
420
  }
363
421
  const rootTextFallback = rootText;
422
+ const point = (v) => Array.isArray(v) && v.length === 2 && typeof v[0] === "number" && Number.isFinite(v[0]) && typeof v[1] === "number" && Number.isFinite(v[1]) ? [v[0], v[1]] : null;
423
+ function cornersOf(v, style, tag) {
424
+ if (v === undefined)
425
+ return undefined;
426
+ const list = Array.isArray(v) ? v : [];
427
+ const [a, b, c, d] = list.map(point);
428
+ if (list.length !== 4 || !a || !b || !c || !d)
429
+ throw new StyleError(`<${tag}> corners: expected four [x, y] points, top left, top right, bottom right, bottom left; got ${JSON.stringify(v)}.`);
430
+ if (style.transform.turn !== 0 || style.transform.tilt !== 0)
431
+ throw new StyleError(`<${tag}> has corners and turn or tilt: corners already say where it lands, so remove turn and tilt.`);
432
+ return [a, b, c, d];
433
+ }
434
+ const CAPS = ["butt", "round", "square"];
435
+ const JOINS = ["miter", "round", "bevel"];
436
+ function oneOf(v, options, fallback, what) {
437
+ if (v === undefined)
438
+ return fallback;
439
+ const found = options.find((o) => o === v);
440
+ if (found === undefined)
441
+ throw new StyleError(`<path> ${what}: expected one of ${options.join(", ")}, got ${JSON.stringify(v)}.`);
442
+ return found;
443
+ }
444
+ function trimOf(v) {
445
+ if (v === undefined)
446
+ return [0, 1];
447
+ if (typeof v === "number" && Number.isFinite(v))
448
+ return [0, v];
449
+ if (Array.isArray(v) && v.length === 2 && typeof v[0] === "number" && Number.isFinite(v[0]) && typeof v[1] === "number" && Number.isFinite(v[1]))
450
+ return [v[0], v[1]];
451
+ throw new StyleError(`<path> trim: expected a fraction or [from, to], got ${JSON.stringify(v)}.`);
452
+ }
453
+ /** One `<path>`, its props read once. Paths aren't styled elements, so this is their boundary. */
454
+ function pathOf(props, text) {
455
+ const d = props["d"], fill = props["fill"], rule = props["fillRule"], stroke = props["stroke"], width = props["strokeWidth"];
456
+ if (typeof d !== "string")
457
+ throw new StyleError("<path> needs d.");
458
+ if (stroke !== undefined && typeof stroke !== "string")
459
+ throw new StyleError("<path> stroke: expected a color.");
460
+ if (width !== undefined && (typeof width !== "number" || !Number.isFinite(width) || width < 0))
461
+ throw new StyleError(`<path> strokeWidth: expected a number of at least 0, got ${JSON.stringify(width)}.`);
462
+ if (props["trim"] !== undefined && stroke === undefined)
463
+ throw new StyleError("<path> trim needs a stroke: it says how much of the outline to draw.");
464
+ return {
465
+ verbs: parsePath(d),
466
+ fill: fill === "none" ? null : typeof fill === "string" ? parseColor(fill) : stroke === undefined ? text.color : null,
467
+ rule: rule === "evenodd" ? "evenodd" : "nonzero",
468
+ stroke: stroke === undefined ? null : {
469
+ color: parseColor(stroke),
470
+ width: width ?? 1,
471
+ cap: oneOf(props["strokeCap"], CAPS, "butt", "strokeCap"),
472
+ join: oneOf(props["strokeJoin"], JOINS, "miter", "strokeJoin"),
473
+ trim: trimOf(props["trim"]),
474
+ },
475
+ };
476
+ }
364
477
  function textOf(n) {
365
478
  switch (n.kind) {
366
479
  case "element":
@@ -1,6 +1,7 @@
1
1
  import type { FullPose, ImageAsset, VideoAsset } from "@tongue-core/film/model";
2
2
  import type { PathVerb } from "../text/font-types.js";
3
3
  import type { Color, Computed, Override, TextStyle } from "../style/computed.js";
4
+ import type { StrokeCap, StrokeJoin } from "../paint/stroke.js";
4
5
  export interface Owner {
5
6
  readonly id: string | null;
6
7
  readonly morph: string | null;
@@ -21,10 +22,19 @@ export interface SayState {
21
22
  /** Pose of each word, in order; words past the end of the list are at rest. */
22
23
  readonly words: ReadonlyArray<FullPose>;
23
24
  }
25
+ export interface SvgStroke {
26
+ readonly color: Color;
27
+ readonly width: number;
28
+ readonly cap: StrokeCap;
29
+ readonly join: StrokeJoin;
30
+ /** The stretch of the outline to draw, as fractions of its length. */
31
+ readonly trim: readonly [number, number];
32
+ }
24
33
  export interface SvgPath {
25
34
  readonly verbs: ReadonlyArray<PathVerb>;
26
- readonly fill: Color;
35
+ readonly fill: Color | null;
27
36
  readonly rule: "nonzero" | "evenodd";
37
+ readonly stroke: SvgStroke | null;
28
38
  }
29
39
  /** What an image box shows: a photo, or one frame of a video. */
30
40
  export type Picture = {
@@ -68,4 +78,17 @@ export interface Box {
68
78
  /** Fraction of its laid-out height the box takes (Show `expand`); below 1 it clips. */
69
79
  readonly grow: number;
70
80
  readonly content: Content;
81
+ /** Drawn onto this four-sided shape, in its parent's coordinates. */
82
+ readonly corners?: Quad;
83
+ /** Paints its children as a group instead of one after another. */
84
+ readonly group?: BoxGroup;
71
85
  }
86
+ export type Quad = readonly [readonly [number, number], readonly [number, number], readonly [number, number], readonly [number, number]];
87
+ export type BoxGroup = {
88
+ readonly kind: "mask";
89
+ readonly invert: boolean;
90
+ }
91
+ /** Motion blur: its children are the same content at different moments, averaged. */
92
+ | {
93
+ readonly kind: "samples";
94
+ };
package/dist/index.d.ts CHANGED
@@ -1,7 +1,8 @@
1
- export { FilmRenderer, sizeAtScale, type FrameOptions, type FrameSize, type DrawnFrame, type ImageUpload } from "./render/renderer.js";
1
+ export { FilmRenderer, type FrameOptions, type DrawnFrame, type ImageUpload } from "./render/renderer.js";
2
+ export { sizeAtScale, type FrameSize } from "./render/size.js";
2
3
  export { collectSounds, audioClips } from "./render/sounds.js";
3
- export { mix, mixWindow, SAMPLE_RATE, SOUND_SECONDS, type MixInput, type WindowInput, type ClipAudio } from "./audio/mix.js";
4
- export { loadAssets, AssetShelf, imageOf, videoOf, audioOf, parseVideoInfo, parseProxyInfo, qoiSize, imageSize, AssetError, type LoadedAssets, type LoadedImage, type LoadedVideo, type VideoInfo, type ProxyInfo, type Fetcher, type LoadOptions } from "./assets/store.js";
4
+ export { mix, mixWindow, busWindow, limitAudio, SAMPLE_RATE, SOUND_SECONDS, type MixInput, type WindowInput, type ClipAudio } from "./audio/mix.js";
5
+ export { loadAssets, AssetShelf, ImageNotLoaded, imageOf, videoOf, audioOf, parseVideoInfo, parseProxyInfo, qoiSize, imageSize, AssetError, type LoadedAssets, type LoadedImage, type LoadedVideo, type VideoInfo, type ProxyInfo, type Fetcher, type LoadOptions } from "./assets/store.js";
5
6
  export { importedPath, importedFiles, videoInfoPath, videoPosterPath, videoProxyDir, videoProxyPath, PROXY_SEGMENT_SECONDS, PROXY_KEYFRAME_SECONDS, type AssetKey } from "./assets/paths.js";
6
7
  export { demuxMp4, sampleData, Mp4Error, type Mp4Video, type Mp4Sample } from "./media/mp4.js";
7
8
  export { yuvToRgba, yuvSize, packedPlanes, encodeYuvImage, isYuvImage, yuvImageSize, yuvImageLayout, writeYuvHeader, YUV_HEADER, type YuvFrame, type YuvFormat, type YuvPlane } from "./media/yuv.js";
@@ -15,7 +16,7 @@ export { decodePng } from "./assets/codecs/png.js";
15
16
  export { decodeJpeg } from "./assets/codecs/jpeg.js";
16
17
  export { encodeQoi, decodeQoi } from "./assets/codecs/qoi.js";
17
18
  export { resizeImage } from "./assets/codecs/resize.js";
18
- export { readWav, writeWav } from "./assets/codecs/wav.js";
19
+ export { readWav, writeWav, writeFloatWav } from "./assets/codecs/wav.js";
19
20
  export { woff2ToSfnt } from "./text/woff2.js";
20
21
  export { parseFont } from "./text/ttf.js";
21
22
  export type { RgbaImage, PcmAudio } from "./assets/codecs/types.js";
package/dist/index.js CHANGED
@@ -1,10 +1,11 @@
1
1
  // @tongue-core/engine: turns a planned film (from @tongue-core/film) into pixels and sound. Evaluate →
2
2
  // layout → paint → rasterize, deterministic and plain TypeScript (plus an embedded WebAssembly
3
3
  // raster), so the same frame is byte-identical in Bun, browsers and Workers.
4
- export { FilmRenderer, sizeAtScale } from "./render/renderer.js";
4
+ export { FilmRenderer } from "./render/renderer.js";
5
+ export { sizeAtScale } from "./render/size.js";
5
6
  export { collectSounds, audioClips } from "./render/sounds.js";
6
- export { mix, mixWindow, SAMPLE_RATE, SOUND_SECONDS } from "./audio/mix.js";
7
- export { loadAssets, AssetShelf, imageOf, videoOf, audioOf, parseVideoInfo, parseProxyInfo, qoiSize, imageSize, AssetError } from "./assets/store.js";
7
+ export { mix, mixWindow, busWindow, limitAudio, SAMPLE_RATE, SOUND_SECONDS } from "./audio/mix.js";
8
+ export { loadAssets, AssetShelf, ImageNotLoaded, imageOf, videoOf, audioOf, parseVideoInfo, parseProxyInfo, qoiSize, imageSize, AssetError } from "./assets/store.js";
8
9
  export { importedPath, importedFiles, videoInfoPath, videoPosterPath, videoProxyDir, videoProxyPath, PROXY_SEGMENT_SECONDS, PROXY_KEYFRAME_SECONDS } from "./assets/paths.js";
9
10
  export { demuxMp4, sampleData, Mp4Error } from "./media/mp4.js";
10
11
  export { yuvToRgba, yuvSize, packedPlanes, encodeYuvImage, isYuvImage, yuvImageSize, yuvImageLayout, writeYuvHeader, YUV_HEADER } from "./media/yuv.js";
@@ -16,6 +17,6 @@ export { decodePng } from "./assets/codecs/png.js";
16
17
  export { decodeJpeg } from "./assets/codecs/jpeg.js";
17
18
  export { encodeQoi, decodeQoi } from "./assets/codecs/qoi.js";
18
19
  export { resizeImage } from "./assets/codecs/resize.js";
19
- export { readWav, writeWav } from "./assets/codecs/wav.js";
20
+ export { readWav, writeWav, writeFloatWav } from "./assets/codecs/wav.js";
20
21
  export { woff2ToSfnt } from "./text/woff2.js";
21
22
  export { parseFont } from "./text/ttf.js";