@tongue-core/sdk 0.0.0-stage → 0.1.0-rc.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +343 -2
  3. package/dist/TongueServices.d.ts +5 -0
  4. package/dist/TongueServices.js +2 -0
  5. package/dist/backends.d.ts +24 -0
  6. package/dist/backends.js +17 -0
  7. package/dist/build/build.d.ts +51 -0
  8. package/dist/build/build.js +175 -0
  9. package/dist/check.d.ts +19 -0
  10. package/dist/check.js +18 -0
  11. package/dist/client.d.ts +47 -0
  12. package/dist/client.js +38 -0
  13. package/dist/encode/encoders.d.ts +68 -0
  14. package/dist/encode/encoders.js +158 -0
  15. package/dist/errors.d.ts +65 -0
  16. package/dist/errors.js +82 -0
  17. package/dist/guards.d.ts +1 -0
  18. package/dist/guards.js +3 -0
  19. package/dist/import/convert.d.ts +12 -0
  20. package/dist/import/convert.js +35 -0
  21. package/dist/import/importer.d.ts +26 -0
  22. package/dist/import/importer.js +30 -0
  23. package/dist/import/local.d.ts +17 -0
  24. package/dist/import/local.js +97 -0
  25. package/dist/import/media.d.ts +52 -0
  26. package/dist/import/media.js +129 -0
  27. package/dist/import/types.d.ts +10 -0
  28. package/dist/import/types.js +1 -0
  29. package/dist/index.d.ts +26 -0
  30. package/dist/index.js +29 -0
  31. package/dist/init.d.ts +30 -0
  32. package/dist/init.js +44 -0
  33. package/dist/media/cache.d.ts +40 -0
  34. package/dist/media/cache.js +161 -0
  35. package/dist/media/prepare.d.ts +44 -0
  36. package/dist/media/prepare.js +206 -0
  37. package/dist/media/snapshot.d.ts +16 -0
  38. package/dist/media/snapshot.js +38 -0
  39. package/dist/points.d.ts +28 -0
  40. package/dist/points.js +76 -0
  41. package/dist/project.d.ts +77 -0
  42. package/dist/project.js +196 -0
  43. package/dist/raster/direct.d.ts +10 -0
  44. package/dist/raster/direct.js +3 -0
  45. package/dist/raster/gpu.d.ts +33 -0
  46. package/dist/raster/gpu.js +112 -0
  47. package/dist/raster/index.d.ts +4 -0
  48. package/dist/raster/index.js +7 -0
  49. package/dist/raster/library.d.ts +9 -0
  50. package/dist/raster/library.js +47 -0
  51. package/dist/raster/native.d.ts +37 -0
  52. package/dist/raster/native.js +128 -0
  53. package/dist/raster/rasters.d.ts +20 -0
  54. package/dist/raster/rasters.js +37 -0
  55. package/dist/render/budget.d.ts +45 -0
  56. package/dist/render/budget.js +83 -0
  57. package/dist/render/entries.d.ts +2 -0
  58. package/dist/render/entries.js +7 -0
  59. package/dist/render/events.d.ts +107 -0
  60. package/dist/render/events.js +1 -0
  61. package/dist/render/film-loader.d.ts +20 -0
  62. package/dist/render/film-loader.js +111 -0
  63. package/dist/render/frame-pipe.d.ts +18 -0
  64. package/dist/render/frame-pipe.js +46 -0
  65. package/dist/render/frame.d.ts +14 -0
  66. package/dist/render/frame.js +33 -0
  67. package/dist/render/inputs.d.ts +5 -0
  68. package/dist/render/inputs.js +36 -0
  69. package/dist/render/pipeline.d.ts +51 -0
  70. package/dist/render/pipeline.js +222 -0
  71. package/dist/render/protocol.d.ts +138 -0
  72. package/dist/render/protocol.js +210 -0
  73. package/dist/render/runs.d.ts +10 -0
  74. package/dist/render/runs.js +42 -0
  75. package/dist/render/soundtrack-process.d.ts +1 -0
  76. package/dist/render/soundtrack-process.js +81 -0
  77. package/dist/render/stills.d.ts +30 -0
  78. package/dist/render/stills.js +73 -0
  79. package/dist/render/stopwatch.d.ts +10 -0
  80. package/dist/render/stopwatch.js +16 -0
  81. package/dist/render/video-frames.d.ts +46 -0
  82. package/dist/render/video-frames.js +256 -0
  83. package/dist/render/worker.d.ts +1 -0
  84. package/dist/render/worker.js +105 -0
  85. package/dist/sound/AudioProcessor.d.ts +59 -0
  86. package/dist/sound/AudioProcessor.js +18 -0
  87. package/dist/sound/Sound.d.ts +48 -0
  88. package/dist/sound/Sound.js +92 -0
  89. package/dist/sound/soundtrack.d.ts +21 -0
  90. package/dist/sound/soundtrack.js +141 -0
  91. package/dist/studio/server.d.ts +19 -0
  92. package/dist/studio/server.js +175 -0
  93. package/dist/tools.d.ts +21 -0
  94. package/dist/tools.js +42 -0
  95. package/dist/version.d.ts +2 -0
  96. package/dist/version.js +7 -0
  97. package/package.json +53 -4
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Backpine
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md CHANGED
@@ -1,3 +1,344 @@
1
- # Temporary Holding Version
1
+ # @tongue-core/sdk
2
2
 
3
- This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
3
+ Everything you do with a Tongue film, as a library: import its assets, plan and check it, draw
4
+ frames, render video, open it in the studio, and list what the machine can render with. The
5
+ `tongue` CLI is a thin shell over this package, so anything the CLI does an application can do
6
+ here, with the same results. Built with Effect v4 (orchestration only: the per-frame path is plain
7
+ TypeScript); a Promise facade is included for applications that don't use Effect.
8
+
9
+ ```bash
10
+ bun add @tongue-core/sdk@next effect@4.0.6 # Bun ≥ 1.3; ffmpeg and ffprobe on the PATH
11
+ ```
12
+
13
+ `effect` is a peer dependency: your application and the SDK share one copy of it, so the SDK's
14
+ services, errors and Layers are the same values as yours.
15
+
16
+ ## Promise API
17
+
18
+ ```ts
19
+ import { createTongue } from "@tongue-core/sdk";
20
+
21
+ const tongue = createTongue();
22
+ await tongue.init("my-film", { template: "social-post" }); // a film project from a template
23
+ const { build } = await tongue.build("my-film"); // my-film/dist: film.json, build.json, bundles
24
+ await tongue.stills("my-film", { frames: [0, 30], outDir: "out/hook", scene: { id: "hook" } }); // one scene alone
25
+ const { text, summary } = await tongue.info("src/film.tsx"); // the timeline, as text and JSON
26
+ const report = await tongue.check("src/film.tsx"); // { ok, problems, warnings }: never rejects for a broken film
27
+ await tongue.sheet("src/film.tsx", { frames: [0, 90, 300], outDir: "out/look" });
28
+ const video = await tongue.render("src/film.tsx", { out: "out/film.mp4", from: 90, to: 300, scale: 0.5 });
29
+ const machine = await tongue.backends(); // rasters and encoders that work here
30
+ ```
31
+
32
+ Failures reject with the SDK's tagged errors (below).
33
+
34
+ ## Effect API
35
+
36
+ ```ts
37
+ import { openFilm, renderVideo, TongueServices } from "@tongue-core/sdk";
38
+ import { Effect } from "effect";
39
+
40
+ const program = Effect.gen(function* () {
41
+ const film = yield* openFilm("src/film.tsx"); // load, import assets, plan
42
+ return yield* renderVideo(film, { out: "out/film.mp4", encoder: "auto", raster: "auto" });
43
+ });
44
+ await Effect.runPromise(program.pipe(Effect.provide(TongueServices.layer)));
45
+ ```
46
+
47
+ `TongueServices.layer` is what every SDK function runs on (the file system, paths and child
48
+ processes, from Bun): provide it once, where your application runs its Effects.
49
+
50
+ | Function | Does |
51
+ |---|---|
52
+ | `initProject(dir, { template, name })` | a film project in a new or empty folder, from a built-in template or a template folder |
53
+ | `listTemplates()` / `readTemplate(nameOrPath)` | the built-in templates, or one by name or path |
54
+ | `buildFilm(project, { out, media })` | builds a film project into `out` (default `dist/`): build.json, film.json, a bundle per scene; `media` configures acquisition and hosted previews |
55
+ | `openFilm(path, { scene, sources, media })` | a film ready to render: a project is built (into `.tongue/builds/<id>`, named by its content) and planned from film.json, a build folder (what `buildFilm` writes) is opened as it is, a film module is imported and planned; assets imported either way. `scene: { id, props }` opens one scene of a project on its own; `sources` overrides video originals; `media` configures hosted-media acquisition and caching ([Footage](#footage)) |
56
+ | `describeFilm(film)` | the plan as text (for agents to read) and as a JSON summary |
57
+ | `checkFilm(path)` | problems and warnings as data |
58
+ | `renderStills(film, { frames, scale, outDir })` / `renderSheet(…)` | PNGs drawn by the wasm raster (the player's), or tiled into one labelled sheet |
59
+ | `renderVideo(film, options)` | an MP4 with sound: the whole film or `[from, to)`, at any scale (`out` defaults to `out/<name>.mp4`); `sound` runs [sound processing](#sound-processing-extensions); `onStart` once the request is checked and the raster, encoder and workers are chosen (`budget` says why that many workers: one per physical core, fewer when memory is short), `onProgress` while it draws. A frame that throws fails the render with `Frame <n>: <message>`. The result's `profile` says where the time went: milliseconds per stage of drawing, decoder starts, pieces encoded, and the wall time of each phase |
60
+ | `renderVideoEvents(film, options)` | the same render as a `Stream` of `RenderEvent`s: `Started`, `Frames`, `Sound` (each processor's progress), `Joining`, then `Done` with the result |
61
+ | `parsePoint`, `parseFrame`, `parseRange`, `clipRange`, `spreadFrames`, `everyFrames` | points in a film as people write them (`90`, `3s`, `50%`, `end`, `site`, `site+30`), for a command line or form of your own; bad ones fail with `InvalidRequest` saying how to write them |
62
+ | `videoInputs(film)`, `checkVideoInputs(film)` | where each video is decoded from, and a check that every one is there |
63
+ | `listBackends()` | every raster and encoder, each tried for real |
64
+ | `studioServer(path, { port, remote })` | the studio on a project (rebuilt and reloaded on every edit) or on a build served anywhere |
65
+
66
+ ## Films and projects
67
+
68
+ A **film project** is a folder with a `tongue.json` ([docs/film-json.md](../../docs/film-json.md)):
69
+ `film.tsx`, `scenes/`, `kit/`, `lib/`, `assets.ts`, `assets/`. It needs nothing installed: the SDK
70
+ reads its files, compiles and bundles it with `@tongue-core/project`, loads the bundles with its own
71
+ `@tongue-core/film`, imports the assets they declare (ffprobe gives media lengths), extracts film.json
72
+ and writes the build. Everything renders from the build: in this process, in the render workers
73
+ (they open build.json and the bundles), and in the studio (`tongue studio` serves the build; the
74
+ browser fetches it). `.tongue/types/film` links to the SDK's `@tongue-core/film`, which the
75
+ project's `tsconfig.json` points editors at.
76
+
77
+ A **film module** (the way films were written before projects) is a `film.tsx` in a folder with a
78
+ `package.json`. The SDK bundles it with Bun (into `.tongue/modules/`, JSX compiled for Tongue) against
79
+ its own `@tongue-core/film`, the copy the engine reads, as it links a project's bundles, then plans
80
+ it. The folder may install `@tongue-core/film` too, for its editor's types
81
+ (`"jsx": "react-jsx", "jsxImportSource": "@tongue-core/film"` in its tsconfig); that copy isn't the one
82
+ that runs.
83
+
84
+ Local asset paths are relative to the root. Tongue writes `.tongue/` there for builds and render
85
+ work, and renders to `out/`. Local converted assets use the project's cache; hosted-media
86
+ downloads and preparation use the shared cache outside the project.
87
+
88
+ A film's assets are what its modules declare (`font()`, `image()`, `audio()`, `video()`) while
89
+ the SDK loads it, plus the media its clips play; `<Film assets={[…]}>` replaces the declared ones.
90
+ Importing turns them into what the engine reads: TrueType fonts, QOI images, WAV audio, video facts
91
+ and a poster frame.
92
+
93
+ ## Footage
94
+
95
+ Hosted URLs are the normal source for application-owned media. Declare them with `video`, `audio`,
96
+ `image` or `font`, with an explicit name and a SHA-256 when the application knows it. The SDK
97
+ streams downloads into a shared cache outside the project, verifies content, and atomically
98
+ promotes completed files. Concurrent requests for the same pinned content share a download.
99
+ Unpinned URLs are revalidated with ETag/Last-Modified when available. Temporary network failures,
100
+ 408, 429 and server errors retry twice by default, with bounded backoff and Retry-After support;
101
+ missing files, denied access and integrity failures are reported as `MediaFailure`.
102
+
103
+ ```ts
104
+ const tongue = createTongue({
105
+ media: {
106
+ cacheDir: "/var/cache/tongue/media",
107
+ store: { directory: "/srv/media", baseUrl: "https://media.example/prepared/" },
108
+ },
109
+ });
110
+ await tongue.build("my-film");
111
+ await tongue.render("my-film/dist", { out: "out/film.mp4" });
112
+ ```
113
+
114
+ A portable build contains manifests and scene bundles, with media locations and hashes in
115
+ `build.json` version 2. Originals and generated remote media are never copied into it. Local
116
+ assets can still be packaged, and version 1 builds remain readable. The host must expose the
117
+ configured store at its `baseUrl` (with CORS for browser clients). The CLI reads the equivalent
118
+ `TONGUE_MEDIA_DIR`, `TONGUE_MEDIA_URL` and `TONGUE_MEDIA_CACHE` environment variables. The default
119
+ cache is `$XDG_CACHE_HOME/tongue/media` or `~/.cache/tongue/media`. `tongue studio` supplies a
120
+ local media store and serves it automatically.
121
+
122
+ For object storage, provide `media.publish(upload)`, an Effect returning the uploaded file's
123
+ HTTP(S) URL. `upload` holds the local `file`, `hash`, `bytes` and `contentType`. Storage belongs
124
+ to the application. A publish callback must finish storing the bytes before returning its URL.
125
+ Use the hash as an idempotent storage key and return a stable URL, so projects share uploaded
126
+ bytes. The built-in directory store does this automatically. Invalid publisher locations fail
127
+ before a portable build is written.
128
+
129
+ Applications can prepare media at ingestion: `prepareMedia(asset, mediaOptions)` returns a
130
+ `tongue.media` manifest with a canonical asset name, import recipe, source hash, duration and
131
+ prepared files. Host that JSON and pass its URL as the declaration's `prepared` option. The same
132
+ preparation is shared across projects and asset names. Preparation recipes include sizing and
133
+ format versions, so incompatible results are rebuilt. Audio previews are sample-indexed WAV
134
+ segments; video previews use the existing segmented H.264 format. Full-quality decoded audio
135
+ remains available to final renders.
136
+
137
+ `media.request(url, signal, headers)` can add authentication or refresh a signed URL without
138
+ serializing credentials into the build. Preserve the supplied signal and conditional headers.
139
+ `retries`, `retryDelayMs`, `timeoutMs`, `concurrency` and `onProgress` control acquisition. Original
140
+ downloads default to a 30-minute timeout; metadata/preview requests default to two minutes.
141
+ Cache contents are shared and host-managed; changing the cache directory isolates a job.
142
+
143
+ Final rendering acquires and verifies its video originals before starting any frame worker.
144
+ Converted assets are also acquired before use. Workers decode local files, with no networking
145
+ in the per-frame path. A fully cached pinned build can render with networking disabled. Build
146
+ snapshots retain immutable prepared-file bindings when a later version changes its assets.
147
+
148
+ A render decodes every video from its **original file**: a build holds only a video's facts, a
149
+ poster and a preview-quality proxy (what the studio plays). Before a frame is drawn, `renderVideo`
150
+ and `renderStills` check that each video's file is there and fail with `AssetFailure` naming the
151
+ ones that aren't; a frame ffmpeg can't decode, or a file shorter than its import counted, fails the
152
+ render with `Frame <n>: Video "<name>": …` and ffmpeg's reason. A poster never stands in for
153
+ footage in a render (that is the player's fallback while frames arrive).
154
+
155
+ A local video is decoded from its source path under the film's root, unless `sources` says otherwise:
156
+
157
+ ```ts
158
+ const film = yield* openFilm("renders/1234/dist", { // a build, without the project around it
159
+ sources: { screen: "/data/media/screen.mp4", camera: "/data/media/camera.mp4" },
160
+ });
161
+ ```
162
+
163
+ A build inside a project (`my-film/dist`) finds local footage in the project; a build on its own
164
+ needs `sources` for local videos that aren't beside it. Remote originals are resolved automatically
165
+ from the pinned registry in the manifest.
166
+
167
+ ## Sound processing (extensions)
168
+
169
+ A render can put part of its soundtrack through processing of your own (voice enhancement, noise
170
+ removal, levelling, effects) and still produce one MP4, in sync. The SDK defines the contract and
171
+ runs it; what a processor does and what it needs (a model, a native tool, a service) is yours, so
172
+ nobody who doesn't use it installs it.
173
+
174
+ ```ts
175
+ import { AudioProcessor, openFilm, renderVideo, Sound } from "@tongue-core/sdk";
176
+
177
+ const enhance = yield* enhanceVoice({ peakDb: -3 }); // an AudioProcessor (below)
178
+ yield* renderVideo(film, {
179
+ sound: [
180
+ Sound.track("voiceover", enhance), // the clips of the track with id "voiceover"
181
+ Sound.track("voiceover", level), // the same selection again: runs after enhance
182
+ Sound.master(loudness), // then the whole mix
183
+ ],
184
+ });
185
+ ```
186
+
187
+ **What a rule takes.** `Sound.track(id, p)`; `Sound.stem({ clips: ["a", "b"] }, p)`;
188
+ `Sound.stem({ voice: true }, p)` (every narration clip); `Sound.master(p)` (the whole mix, after
189
+ the stems, before the limiter). A selection is data (`Sound.AudioSelection` decodes it from flags or
190
+ JSON). The selected clips are mixed into a **stem** of their own (their trims, speed, gain and fades
191
+ applied) across the whole film, processed, and added back; everything else sounds as it always has.
192
+ A track or clip the film doesn't have, or a visual track, is an `InvalidRequest` listing what it
193
+ does have; a clip goes into one stem, so overlapping selections are refused unless they are the same
194
+ (then their processors run in order).
195
+
196
+ **The contract.** A processor gets a `Stem`: `samples` per channel (the whole film at 48 kHz, from
197
+ its first frame), `channels` (2), the stem as a float WAV `file` and as decoded `pcm`, the `clips` in
198
+ it and the `plan`, a `workDir`, and `progress(0..1)`. It returns `{ pcm }` or `{ file }`, plus an
199
+ optional `report` (any JSON). What it returns must have exactly the stem's rate, channels and
200
+ length, and only finite samples, or the render fails with `ProcessorFailure`: that is what keeps
201
+ the sound in sync, so a processor with a delay (a model's lookahead) takes it out itself. Any
202
+ failure of the processor, or a defect in it, is a `ProcessorFailure` naming it.
203
+
204
+ **Two ways to make one.** `AudioProcessor.fromPcm({ key, process: (pcm, stem) => pcm })` for
205
+ processing that needs only the samples. `AudioProcessor.make({ key, process: (stem) => Effect })`
206
+ for anything else: the services `process` uses are taken from where `make` runs and travel with the
207
+ processor, so the render itself never needs them:
208
+
209
+ ```ts
210
+ // voice-enhance: its own package, depending on @tongue-core/sdk and on what it needs.
211
+ interface DenoiserOperations {
212
+ readonly name: string;
213
+ readonly denoise: (channel: Float32Array, sampleRate: number) => Effect.Effect<Float32Array>;
214
+ }
215
+ export class Denoiser extends Context.Service<Denoiser, DenoiserOperations>()("voice-enhance/Denoiser") {
216
+ static readonly layer = Layer.effect(Denoiser, loadModel); // the heavy part, paid once, only by those who provide it
217
+ }
218
+
219
+ export const enhanceVoice = (settings: EnhanceSettings) =>
220
+ AudioProcessor.make({
221
+ key: `voice-enhance/v1 peak ${settings.peakDb} dB`,
222
+ process: (stem) => Effect.gen(function* () {
223
+ const denoiser = yield* Denoiser;
224
+ const pcm = yield* stem.pcm;
225
+ const channels = yield* Effect.forEach(pcm.channels, (c) => denoiser.denoise(c, pcm.sampleRate));
226
+ yield* stem.progress(1);
227
+ return { pcm: { sampleRate: pcm.sampleRate, channels }, report: { denoiser: denoiser.name } };
228
+ }),
229
+ }); // Effect<AudioProcessor, never, Denoiser>
230
+
231
+ // In your command line: make the processor where its Layer is provided.
232
+ const render = Command.make("render", { project: Argument.String("project"), enhance: Flag.String("enhance").pipe(Flag.atLeast(0)) },
233
+ Effect.fn(function* (flags) {
234
+ const film = yield* openFilm(flags.project);
235
+ const enhance = yield* enhanceVoice({ peakDb: -3 });
236
+ return yield* renderVideo(film, { sound: flags.enhance.map((track) => Sound.track(track, enhance)) });
237
+ }),
238
+ ).pipe(Command.provide(Denoiser.layer));
239
+ ```
240
+
241
+ **The key and the cache.** A processor's `key` names it, its version and its settings. A result is
242
+ cached by the key and the exact audio it was given (`.tongue/cache/sound`, the newest 32 kept): render
243
+ again after changing anything that doesn't change that audio (the picture, another track, the
244
+ range, the scale) and the processor doesn't run; the result says `cached: true`. Change the key
245
+ whenever the output would change.
246
+
247
+ **What happened.** `VideoResult.sound` lists each processor run: the stem, its clips, the
248
+ processor's key, whether it was cached, its seconds and its report. `renderVideoEvents` streams
249
+ `Sound` events (`started`, `progress`, `done`, `cached`) while it runs.
250
+
251
+ **Where it runs.** Processors run in your process, alongside the render workers, while the frames
252
+ are drawn (the soundtrack always has); keep heavy work off its main thread (`effect/workers`, a
253
+ child process), or it slows the hand-off of frames to the workers.
254
+
255
+ ## How a video renders
256
+
257
+ ```
258
+ coordinator ──segments──▶ worker 1: frames ─▶ raster ─▶ encoder ─▶ piece-000000.mp4
259
+ ◀─progress── worker 2 … piece-000280.mp4
260
+ soundtrack: sounds + audio clips ─▶ mix ─▶ AAC audio.m4a
261
+ with sound processing: stems ─▶ processors (here, cached) ─▶ sum ─▶ master ─▶ limiter ─▶ AAC
262
+ join: concat + audio ─(stream copy)─▶ film.mp4, frame count probed
263
+ ```
264
+
265
+ - **Workers** are processes (`src/render/worker.ts`), one per physical core by default, fewer when
266
+ the memory available wouldn't hold that many (`src/render/budget.ts`). Each loads
267
+ the film and its raster once and draws every segment it is sent, so the JIT stays warm and glyphs
268
+ and images stay uploaded.
269
+ - **Segments** are a quarter to half a second of frames: what a worker is given at a time. Each
270
+ worker starts on its own contiguous run of them; one that finishes early takes over the later
271
+ half of the longest run left (`src/render/runs.ts`). So every worker draws long contiguous
272
+ stretches, and its video decoders read straight through instead of seeking.
273
+ - **Pieces** are what is encoded: each contiguous stretch a worker draws goes to one encoder, as one
274
+ closed-GOP file; the join copies streams without re-encoding. Which pieces a render makes depends
275
+ on how the work was shared out; the frames don't.
276
+ - **Video clips** are decoded by ffmpeg from their original files, in order, frame after frame, a
277
+ few frames ahead, through a FIFO that holds ffmpeg back when it gets ahead; restarted on jumps,
278
+ a window at a time for clips playing backwards, and closed when unused. A frame that can't be
279
+ decoded fails the render ([Footage](#footage)).
280
+ - **The soundtrack** is mixed in its own process while the workers draw (`soundtrack-process.ts`);
281
+ with sound processing, it writes the stems, the processors run in the coordinator, and it adds
282
+ them back, limits and encodes (`sound/soundtrack.ts`).
283
+
284
+ Rasters (who draws) and encoders (who encodes) are chosen by name, because the choice is made here
285
+ but runs inside the workers:
286
+
287
+ | Raster | |
288
+ |---|---|
289
+ | `native` | the Rust raster built for x86-64-v3 (`@tongue-core/raster-linux-x64`); byte-identical to wasm, 2–3× faster |
290
+ | `wasm` | the engine's embedded WebAssembly raster, the browser player's; runs anywhere |
291
+ | `gpu` | EGL + GLES 3.1; within a few levels of the CPU rasters, never chosen by `auto` |
292
+
293
+ Encoders: `x264` (veryfast, crf 14), `vaapi-hevc` / `vaapi-h264` (constant QP 18 / 16, with
294
+ CPU-conversion variants), and `nvenc-*` / `qsv-*` (written from vendor documentation, untested).
295
+ `auto` takes the first that works of nvenc-hevc, vaapi-hevc, vaapi-hevc-cpuconv, qsv-hevc, x264.
296
+
297
+ On this machine (16 threads, AMD GPU) the 35.5 s demo renders at 1080p with sound in about 12 s
298
+ (≈ 87 fps) with `native` + `vaapi-hevc`.
299
+
300
+ ## Errors
301
+
302
+ Every failure is a `Schema.TaggedError` with a message that says what to change. `exitCodeOf`
303
+ gives the CLI's exit code.
304
+
305
+ | Error | When | Exit |
306
+ |---|---|---|
307
+ | `FilmFailure` | the module is missing, throws, isn't a `<Film>`, or its timeline doesn't plan | 2 |
308
+ | `AssetFailure` | an asset's file is missing or unreadable, or a video's original file isn't there to render from | 2 |
309
+ | `InvalidRequest` | a frame outside the film, a bad range or scale | 2 |
310
+ | `ToolFailure` | ffmpeg or ffprobe is missing or failed | 3 |
311
+ | `BackendUnavailable` | the raster or encoder asked for doesn't work here | 3 |
312
+ | `RenderFailure` | a worker died, a frame failed, or a video frame couldn't be decoded | 3 |
313
+ | `ProcessorFailure` | a sound processor failed, or returned audio that would break sync (rate, channels, length) or isn't audio | 3 |
314
+ | `LayoutError`, `CompileError`, `LoadError`, `TimelineError`, `DocError`, `TemplateError` | a film project doesn't build (from `@tongue-core/project`: each names its file, line or path) | 2 |
315
+
316
+ ## Environment
317
+
318
+ | Variable | |
319
+ |---|---|
320
+ | `TONGUE_FFMPEG`, `TONGUE_FFPROBE` | the tools' paths (default: `ffmpeg`, `ffprobe` on the PATH) |
321
+ | `TONGUE_NATIVE_RASTER`, `TONGUE_GPU_RASTER` | a raster library to use instead of `@tongue-core/raster-linux-x64`'s |
322
+ | `TONGUE_VAAPI_DEVICE` | the render node for VAAPI and Quick Sync (default `/dev/dri/renderD128`) |
323
+
324
+ ## Layout
325
+
326
+ ```
327
+ src/
328
+ index.ts, client.ts the Effect API and the Promise facade
329
+ project.ts projects, openFilm and buildFilm
330
+ build/build.ts building a film project on this machine: its files, a module host, the build folder, editor types
331
+ init.ts init from templates
332
+ check.ts, backends.ts checkFilm, listBackends
333
+ import/ asset import: fonts, images, audio, video (ffprobe, posters)
334
+ render/ stills, the video pipeline and its events, its worker and soundtrack processes, video decoding and inputs
335
+ sound/ sound processing: Sound (rules), AudioProcessor (the contract), the soundtrack around them
336
+ points.ts points and ranges in a film, as people write them
337
+ TongueServices.ts the layer the SDK runs on
338
+ encode/encoders.ts the ffmpeg encoders and how each is tried
339
+ raster/ the native and GPU raster bindings (index.ts) and the registry (rasters.ts)
340
+ studio/server.ts the studio's server: the app bundled, a project's build served and rebuilt
341
+ test/ unit tests, end-to-end ones on fixtures/tiny (a film with text, video and sound; sound
342
+ processing and footage that can't be decoded on copies of it), builds rendered with `sources`, and
343
+ projects from every template (project.test.ts: film.json draws the same frames as film.tsx)
344
+ ```
@@ -0,0 +1,5 @@
1
+ import { BunServices } from "@effect/platform-bun";
2
+ import type { Layer } from "effect";
3
+ /** The services every SDK function needs. */
4
+ export type TongueServices = BunServices.BunServices;
5
+ export declare const layer: Layer.Layer<TongueServices>;
@@ -0,0 +1,2 @@
1
+ import { BunServices } from "@effect/platform-bun";
2
+ export const layer = BunServices.layer;
@@ -0,0 +1,24 @@
1
+ import { Effect } from "effect";
2
+ import { type EncoderName } from "./encode/encoders.js";
3
+ import { type RasterName } from "./raster/rasters.js";
4
+ export interface BackendsReport {
5
+ readonly rasters: ReadonlyArray<{
6
+ readonly name: RasterName;
7
+ readonly available: boolean;
8
+ readonly exact: boolean;
9
+ readonly summary: string;
10
+ }>;
11
+ readonly encoders: ReadonlyArray<{
12
+ readonly name: EncoderName;
13
+ readonly available: boolean;
14
+ readonly hardware: boolean;
15
+ readonly tested: boolean;
16
+ readonly summary: string;
17
+ }>;
18
+ /** What `auto` picks here. */
19
+ readonly auto: {
20
+ readonly raster: RasterName;
21
+ readonly encoder: EncoderName | null;
22
+ };
23
+ }
24
+ export declare const listBackends: () => Effect.Effect<BackendsReport, import("effect/Config").ConfigError, never>;
@@ -0,0 +1,17 @@
1
+ import { Effect } from "effect";
2
+ import { AUTO_ENCODERS, DEFAULT_SETTINGS, ENCODER_NAMES, ENCODER_SUMMARIES, encoderSpec, encoderWorks } from "./encode/encoders.js";
3
+ import { autoRaster, RASTERS } from "./raster/rasters.js";
4
+ import { ffmpegPath } from "./tools.js";
5
+ export const listBackends = Effect.fn("listBackends")(function* () {
6
+ const ffmpeg = yield* ffmpegPath;
7
+ const encoders = yield* Effect.forEach(ENCODER_NAMES, (name) => Effect.promise(async () => {
8
+ const spec = encoderSpec(name, DEFAULT_SETTINGS, 30);
9
+ return { name, available: await encoderWorks(ffmpeg, spec), hardware: spec.hardware, tested: spec.tested, summary: ENCODER_SUMMARIES[name] };
10
+ }), { concurrency: "unbounded" });
11
+ const report = {
12
+ rasters: RASTERS.map((r) => ({ name: r.name, available: r.available(), exact: r.exact, summary: r.summary })),
13
+ encoders,
14
+ auto: { raster: autoRaster().name, encoder: AUTO_ENCODERS.find((n) => encoders.some((e) => e.name === n && e.available)) ?? null },
15
+ };
16
+ return report;
17
+ });
@@ -0,0 +1,51 @@
1
+ import type { Asset, MediaInfo, SceneDefinition } from "@tongue-core/film";
2
+ import { type Build, type ModuleHost, type ProjectFiles } from "@tongue-core/project";
3
+ import { Effect, FileSystem, type PlatformError } from "effect";
4
+ import { type ImportReport } from "../import/importer.js";
5
+ import type { Project } from "../project.js";
6
+ import { type MediaOptions } from "../media/cache.js";
7
+ /** A project's code and data: every .ts, .tsx and .json file outside hidden folders and the ones
8
+ * tools write. */
9
+ export declare const readProjectFiles: (root: string) => Effect.Effect<ProjectFiles, PlatformError.PlatformError, FileSystem.FileSystem>;
10
+ /** Loads bundles by writing each to `dir` (named by its hash, so written once) and importing it. */
11
+ export declare const fileModuleHost: (dir: string) => ModuleHost;
12
+ /** Points the project's `.tongue/types/film` at this SDK's @tongue-core/film (tsconfig.json reads it). */
13
+ export declare const linkFilmTypes: (root: string) => Effect.Effect<void, PlatformError.PlatformError, FileSystem.FileSystem>;
14
+ export interface BuiltProject {
15
+ readonly assetsDir: string;
16
+ readonly build: Build;
17
+ /** Where the build was written. */
18
+ readonly dir: string;
19
+ readonly imported: ImportReport;
20
+ /** The scenes the bundles export, by id: what film.json is assembled with. */
21
+ readonly scenes: ReadonlyMap<string, SceneDefinition>;
22
+ readonly media: MediaInfo;
23
+ }
24
+ /** The imported files a film's assets are (a video is its facts, its poster and its proxy's
25
+ * segments), with their sizes and hashes: what build.json lists. */
26
+ export declare const importedFiles: (project: Project, assets: readonly Asset[]) => Effect.Effect<{
27
+ readonly path: string;
28
+ readonly bytes: number;
29
+ readonly hash: string;
30
+ readonly url?: string | undefined;
31
+ }[], PlatformError.PlatformError, FileSystem.FileSystem>;
32
+ /** How a written build reaches its assets: copies of the files (a build to publish), or a link to
33
+ * the project's imported assets (a build for Tongue's own use). */
34
+ export type AssetPlacement = {
35
+ readonly kind: "copy";
36
+ readonly from: string;
37
+ } | {
38
+ readonly kind: "link";
39
+ readonly to: string;
40
+ };
41
+ /** Writes a build to `dir`, replacing the build that was there (old bundles and assets included). */
42
+ export declare const writeBuildFiles: (dir: string, build: Build, assets: AssetPlacement) => Effect.Effect<void, PlatformError.PlatformError, FileSystem.FileSystem>;
43
+ /** Keeps a build in `.tongue/builds/<id>`, named by its content and never rewritten, so renders
44
+ * reading one build are never disturbed by the next. Written to a temporary folder first and moved
45
+ * into place, so a build folder is either whole or not there. */
46
+ export declare const keepBuild: (project: Project, build: Build) => Effect.Effect<string, PlatformError.PlatformError, FileSystem.FileSystem>;
47
+ /** Builds a film project into `dir` (what `tongue build` does: dist/ by default), replacing what
48
+ * was there, with copies of its imported assets: a folder any static host can serve. */
49
+ export declare const buildFilmProject: (project: Project, dir: string, options?: MediaOptions | undefined) => Effect.Effect<BuiltProject, import("../errors.js").AssetFailure | import("../errors.js").ToolFailure | import("@tongue-core/project/media").MediaFailure | import("@tongue-core/project").LayoutError | import("@tongue-core/project").CompileError | import("@tongue-core/project").LoadError | import("@tongue-core/project").TimelineError | import("@tongue-core/project").DocError | import("effect/Config").ConfigError | PlatformError.PlatformError | import("effect/Schema").SchemaError, import("effect/process/ChildProcessSpawner").ChildProcessSpawner | FileSystem.FileSystem | import("effect/Path").Path>;
50
+ /** Builds a film project for Tongue's own use (stills, renders, dev): kept in .tongue/builds. */
51
+ export declare const buildForWork: (project: Project, options?: MediaOptions | undefined) => Effect.Effect<BuiltProject, import("../errors.js").AssetFailure | import("../errors.js").ToolFailure | import("@tongue-core/project/media").MediaFailure | import("@tongue-core/project").LayoutError | import("@tongue-core/project").CompileError | import("@tongue-core/project").LoadError | import("@tongue-core/project").TimelineError | import("@tongue-core/project").DocError | import("effect/Config").ConfigError | PlatformError.PlatformError | import("effect/Schema").SchemaError, import("effect/process/ChildProcessSpawner").ChildProcessSpawner | FileSystem.FileSystem | import("effect/Path").Path>;