@tongue-core/sdk 0.0.0-stage → 0.1.0-rc.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +343 -2
- package/dist/TongueServices.d.ts +5 -0
- package/dist/TongueServices.js +2 -0
- package/dist/backends.d.ts +24 -0
- package/dist/backends.js +17 -0
- package/dist/build/build.d.ts +51 -0
- package/dist/build/build.js +175 -0
- package/dist/check.d.ts +19 -0
- package/dist/check.js +18 -0
- package/dist/client.d.ts +47 -0
- package/dist/client.js +38 -0
- package/dist/encode/encoders.d.ts +68 -0
- package/dist/encode/encoders.js +158 -0
- package/dist/errors.d.ts +65 -0
- package/dist/errors.js +82 -0
- package/dist/guards.d.ts +1 -0
- package/dist/guards.js +3 -0
- package/dist/import/convert.d.ts +12 -0
- package/dist/import/convert.js +35 -0
- package/dist/import/importer.d.ts +26 -0
- package/dist/import/importer.js +30 -0
- package/dist/import/local.d.ts +17 -0
- package/dist/import/local.js +97 -0
- package/dist/import/media.d.ts +52 -0
- package/dist/import/media.js +129 -0
- package/dist/import/types.d.ts +10 -0
- package/dist/import/types.js +1 -0
- package/dist/index.d.ts +26 -0
- package/dist/index.js +29 -0
- package/dist/init.d.ts +30 -0
- package/dist/init.js +44 -0
- package/dist/media/cache.d.ts +40 -0
- package/dist/media/cache.js +161 -0
- package/dist/media/prepare.d.ts +44 -0
- package/dist/media/prepare.js +206 -0
- package/dist/media/snapshot.d.ts +16 -0
- package/dist/media/snapshot.js +38 -0
- package/dist/points.d.ts +28 -0
- package/dist/points.js +76 -0
- package/dist/project.d.ts +77 -0
- package/dist/project.js +196 -0
- package/dist/raster/direct.d.ts +10 -0
- package/dist/raster/direct.js +3 -0
- package/dist/raster/gpu.d.ts +33 -0
- package/dist/raster/gpu.js +112 -0
- package/dist/raster/index.d.ts +4 -0
- package/dist/raster/index.js +7 -0
- package/dist/raster/library.d.ts +9 -0
- package/dist/raster/library.js +47 -0
- package/dist/raster/native.d.ts +37 -0
- package/dist/raster/native.js +128 -0
- package/dist/raster/rasters.d.ts +20 -0
- package/dist/raster/rasters.js +37 -0
- package/dist/render/budget.d.ts +45 -0
- package/dist/render/budget.js +83 -0
- package/dist/render/entries.d.ts +2 -0
- package/dist/render/entries.js +7 -0
- package/dist/render/events.d.ts +107 -0
- package/dist/render/events.js +1 -0
- package/dist/render/film-loader.d.ts +20 -0
- package/dist/render/film-loader.js +111 -0
- package/dist/render/frame-pipe.d.ts +18 -0
- package/dist/render/frame-pipe.js +46 -0
- package/dist/render/frame.d.ts +14 -0
- package/dist/render/frame.js +33 -0
- package/dist/render/inputs.d.ts +5 -0
- package/dist/render/inputs.js +36 -0
- package/dist/render/pipeline.d.ts +51 -0
- package/dist/render/pipeline.js +222 -0
- package/dist/render/protocol.d.ts +138 -0
- package/dist/render/protocol.js +210 -0
- package/dist/render/runs.d.ts +10 -0
- package/dist/render/runs.js +42 -0
- package/dist/render/soundtrack-process.d.ts +1 -0
- package/dist/render/soundtrack-process.js +81 -0
- package/dist/render/stills.d.ts +30 -0
- package/dist/render/stills.js +73 -0
- package/dist/render/stopwatch.d.ts +10 -0
- package/dist/render/stopwatch.js +16 -0
- package/dist/render/video-frames.d.ts +46 -0
- package/dist/render/video-frames.js +256 -0
- package/dist/render/worker.d.ts +1 -0
- package/dist/render/worker.js +105 -0
- package/dist/sound/AudioProcessor.d.ts +59 -0
- package/dist/sound/AudioProcessor.js +18 -0
- package/dist/sound/Sound.d.ts +48 -0
- package/dist/sound/Sound.js +92 -0
- package/dist/sound/soundtrack.d.ts +21 -0
- package/dist/sound/soundtrack.js +141 -0
- package/dist/studio/server.d.ts +19 -0
- package/dist/studio/server.js +175 -0
- package/dist/tools.d.ts +21 -0
- package/dist/tools.js +42 -0
- package/dist/version.d.ts +2 -0
- package/dist/version.js +7 -0
- package/package.json +53 -4
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Backpine
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,344 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @tongue-core/sdk
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Everything you do with a Tongue film, as a library: import its assets, plan and check it, draw
|
|
4
|
+
frames, render video, open it in the studio, and list what the machine can render with. The
|
|
5
|
+
`tongue` CLI is a thin shell over this package, so anything the CLI does an application can do
|
|
6
|
+
here, with the same results. Built with Effect v4 (orchestration only: the per-frame path is plain
|
|
7
|
+
TypeScript); a Promise facade is included for applications that don't use Effect.
|
|
8
|
+
|
|
9
|
+
```bash
|
|
10
|
+
bun add @tongue-core/sdk@next effect@4.0.6 # Bun ≥ 1.3; ffmpeg and ffprobe on the PATH
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
`effect` is a peer dependency: your application and the SDK share one copy of it, so the SDK's
|
|
14
|
+
services, errors and Layers are the same values as yours.
|
|
15
|
+
|
|
16
|
+
## Promise API
|
|
17
|
+
|
|
18
|
+
```ts
|
|
19
|
+
import { createTongue } from "@tongue-core/sdk";
|
|
20
|
+
|
|
21
|
+
const tongue = createTongue();
|
|
22
|
+
await tongue.init("my-film", { template: "social-post" }); // a film project from a template
|
|
23
|
+
const { build } = await tongue.build("my-film"); // my-film/dist: film.json, build.json, bundles
|
|
24
|
+
await tongue.stills("my-film", { frames: [0, 30], outDir: "out/hook", scene: { id: "hook" } }); // one scene alone
|
|
25
|
+
const { text, summary } = await tongue.info("src/film.tsx"); // the timeline, as text and JSON
|
|
26
|
+
const report = await tongue.check("src/film.tsx"); // { ok, problems, warnings }: never rejects for a broken film
|
|
27
|
+
await tongue.sheet("src/film.tsx", { frames: [0, 90, 300], outDir: "out/look" });
|
|
28
|
+
const video = await tongue.render("src/film.tsx", { out: "out/film.mp4", from: 90, to: 300, scale: 0.5 });
|
|
29
|
+
const machine = await tongue.backends(); // rasters and encoders that work here
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
Failures reject with the SDK's tagged errors (below).
|
|
33
|
+
|
|
34
|
+
## Effect API
|
|
35
|
+
|
|
36
|
+
```ts
|
|
37
|
+
import { openFilm, renderVideo, TongueServices } from "@tongue-core/sdk";
|
|
38
|
+
import { Effect } from "effect";
|
|
39
|
+
|
|
40
|
+
const program = Effect.gen(function* () {
|
|
41
|
+
const film = yield* openFilm("src/film.tsx"); // load, import assets, plan
|
|
42
|
+
return yield* renderVideo(film, { out: "out/film.mp4", encoder: "auto", raster: "auto" });
|
|
43
|
+
});
|
|
44
|
+
await Effect.runPromise(program.pipe(Effect.provide(TongueServices.layer)));
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
`TongueServices.layer` is what every SDK function runs on (the file system, paths and child
|
|
48
|
+
processes, from Bun): provide it once, where your application runs its Effects.
|
|
49
|
+
|
|
50
|
+
| Function | Does |
|
|
51
|
+
|---|---|
|
|
52
|
+
| `initProject(dir, { template, name })` | a film project in a new or empty folder, from a built-in template or a template folder |
|
|
53
|
+
| `listTemplates()` / `readTemplate(nameOrPath)` | the built-in templates, or one by name or path |
|
|
54
|
+
| `buildFilm(project, { out, media })` | builds a film project into `out` (default `dist/`): build.json, film.json, a bundle per scene; `media` configures acquisition and hosted previews |
|
|
55
|
+
| `openFilm(path, { scene, sources, media })` | a film ready to render: a project is built (into `.tongue/builds/<id>`, named by its content) and planned from film.json, a build folder (what `buildFilm` writes) is opened as it is, a film module is imported and planned; assets imported either way. `scene: { id, props }` opens one scene of a project on its own; `sources` overrides video originals; `media` configures hosted-media acquisition and caching ([Footage](#footage)) |
|
|
56
|
+
| `describeFilm(film)` | the plan as text (for agents to read) and as a JSON summary |
|
|
57
|
+
| `checkFilm(path)` | problems and warnings as data |
|
|
58
|
+
| `renderStills(film, { frames, scale, outDir })` / `renderSheet(…)` | PNGs drawn by the wasm raster (the player's), or tiled into one labelled sheet |
|
|
59
|
+
| `renderVideo(film, options)` | an MP4 with sound: the whole film or `[from, to)`, at any scale (`out` defaults to `out/<name>.mp4`); `sound` runs [sound processing](#sound-processing-extensions); `onStart` once the request is checked and the raster, encoder and workers are chosen (`budget` says why that many workers: one per physical core, fewer when memory is short), `onProgress` while it draws. A frame that throws fails the render with `Frame <n>: <message>`. The result's `profile` says where the time went: milliseconds per stage of drawing, decoder starts, pieces encoded, and the wall time of each phase |
|
|
60
|
+
| `renderVideoEvents(film, options)` | the same render as a `Stream` of `RenderEvent`s: `Started`, `Frames`, `Sound` (each processor's progress), `Joining`, then `Done` with the result |
|
|
61
|
+
| `parsePoint`, `parseFrame`, `parseRange`, `clipRange`, `spreadFrames`, `everyFrames` | points in a film as people write them (`90`, `3s`, `50%`, `end`, `site`, `site+30`), for a command line or form of your own; bad ones fail with `InvalidRequest` saying how to write them |
|
|
62
|
+
| `videoInputs(film)`, `checkVideoInputs(film)` | where each video is decoded from, and a check that every one is there |
|
|
63
|
+
| `listBackends()` | every raster and encoder, each tried for real |
|
|
64
|
+
| `studioServer(path, { port, remote })` | the studio on a project (rebuilt and reloaded on every edit) or on a build served anywhere |
|
|
65
|
+
|
|
66
|
+
## Films and projects
|
|
67
|
+
|
|
68
|
+
A **film project** is a folder with a `tongue.json` ([docs/film-json.md](../../docs/film-json.md)):
|
|
69
|
+
`film.tsx`, `scenes/`, `kit/`, `lib/`, `assets.ts`, `assets/`. It needs nothing installed: the SDK
|
|
70
|
+
reads its files, compiles and bundles it with `@tongue-core/project`, loads the bundles with its own
|
|
71
|
+
`@tongue-core/film`, imports the assets they declare (ffprobe gives media lengths), extracts film.json
|
|
72
|
+
and writes the build. Everything renders from the build: in this process, in the render workers
|
|
73
|
+
(they open build.json and the bundles), and in the studio (`tongue studio` serves the build; the
|
|
74
|
+
browser fetches it). `.tongue/types/film` links to the SDK's `@tongue-core/film`, which the
|
|
75
|
+
project's `tsconfig.json` points editors at.
|
|
76
|
+
|
|
77
|
+
A **film module** (the way films were written before projects) is a `film.tsx` in a folder with a
|
|
78
|
+
`package.json`. The SDK bundles it with Bun (into `.tongue/modules/`, JSX compiled for Tongue) against
|
|
79
|
+
its own `@tongue-core/film`, the copy the engine reads, as it links a project's bundles, then plans
|
|
80
|
+
it. The folder may install `@tongue-core/film` too, for its editor's types
|
|
81
|
+
(`"jsx": "react-jsx", "jsxImportSource": "@tongue-core/film"` in its tsconfig); that copy isn't the one
|
|
82
|
+
that runs.
|
|
83
|
+
|
|
84
|
+
Local asset paths are relative to the root. Tongue writes `.tongue/` there for builds and render
|
|
85
|
+
work, and renders to `out/`. Local converted assets use the project's cache; hosted-media
|
|
86
|
+
downloads and preparation use the shared cache outside the project.
|
|
87
|
+
|
|
88
|
+
A film's assets are what its modules declare (`font()`, `image()`, `audio()`, `video()`) while
|
|
89
|
+
the SDK loads it, plus the media its clips play; `<Film assets={[…]}>` replaces the declared ones.
|
|
90
|
+
Importing turns them into what the engine reads: TrueType fonts, QOI images, WAV audio, video facts
|
|
91
|
+
and a poster frame.
|
|
92
|
+
|
|
93
|
+
## Footage
|
|
94
|
+
|
|
95
|
+
Hosted URLs are the normal source for application-owned media. Declare them with `video`, `audio`,
|
|
96
|
+
`image` or `font`, with an explicit name and a SHA-256 when the application knows it. The SDK
|
|
97
|
+
streams downloads into a shared cache outside the project, verifies content, and atomically
|
|
98
|
+
promotes completed files. Concurrent requests for the same pinned content share a download.
|
|
99
|
+
Unpinned URLs are revalidated with ETag/Last-Modified when available. Temporary network failures,
|
|
100
|
+
408, 429 and server errors retry twice by default, with bounded backoff and Retry-After support;
|
|
101
|
+
missing files, denied access and integrity failures are reported as `MediaFailure`.
|
|
102
|
+
|
|
103
|
+
```ts
|
|
104
|
+
const tongue = createTongue({
|
|
105
|
+
media: {
|
|
106
|
+
cacheDir: "/var/cache/tongue/media",
|
|
107
|
+
store: { directory: "/srv/media", baseUrl: "https://media.example/prepared/" },
|
|
108
|
+
},
|
|
109
|
+
});
|
|
110
|
+
await tongue.build("my-film");
|
|
111
|
+
await tongue.render("my-film/dist", { out: "out/film.mp4" });
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
A portable build contains manifests and scene bundles, with media locations and hashes in
|
|
115
|
+
`build.json` version 2. Originals and generated remote media are never copied into it. Local
|
|
116
|
+
assets can still be packaged, and version 1 builds remain readable. The host must expose the
|
|
117
|
+
configured store at its `baseUrl` (with CORS for browser clients). The CLI reads the equivalent
|
|
118
|
+
`TONGUE_MEDIA_DIR`, `TONGUE_MEDIA_URL` and `TONGUE_MEDIA_CACHE` environment variables. The default
|
|
119
|
+
cache is `$XDG_CACHE_HOME/tongue/media` or `~/.cache/tongue/media`. `tongue studio` supplies a
|
|
120
|
+
local media store and serves it automatically.
|
|
121
|
+
|
|
122
|
+
For object storage, provide `media.publish(upload)`, an Effect returning the uploaded file's
|
|
123
|
+
HTTP(S) URL. `upload` holds the local `file`, `hash`, `bytes` and `contentType`. Storage belongs
|
|
124
|
+
to the application. A publish callback must finish storing the bytes before returning its URL.
|
|
125
|
+
Use the hash as an idempotent storage key and return a stable URL, so projects share uploaded
|
|
126
|
+
bytes. The built-in directory store does this automatically. Invalid publisher locations fail
|
|
127
|
+
before a portable build is written.
|
|
128
|
+
|
|
129
|
+
Applications can prepare media at ingestion: `prepareMedia(asset, mediaOptions)` returns a
|
|
130
|
+
`tongue.media` manifest with a canonical asset name, import recipe, source hash, duration and
|
|
131
|
+
prepared files. Host that JSON and pass its URL as the declaration's `prepared` option. The same
|
|
132
|
+
preparation is shared across projects and asset names. Preparation recipes include sizing and
|
|
133
|
+
format versions, so incompatible results are rebuilt. Audio previews are sample-indexed WAV
|
|
134
|
+
segments; video previews use the existing segmented H.264 format. Full-quality decoded audio
|
|
135
|
+
remains available to final renders.
|
|
136
|
+
|
|
137
|
+
`media.request(url, signal, headers)` can add authentication or refresh a signed URL without
|
|
138
|
+
serializing credentials into the build. Preserve the supplied signal and conditional headers.
|
|
139
|
+
`retries`, `retryDelayMs`, `timeoutMs`, `concurrency` and `onProgress` control acquisition. Original
|
|
140
|
+
downloads default to a 30-minute timeout; metadata/preview requests default to two minutes.
|
|
141
|
+
Cache contents are shared and host-managed; changing the cache directory isolates a job.
|
|
142
|
+
|
|
143
|
+
Final rendering acquires and verifies its video originals before starting any frame worker.
|
|
144
|
+
Converted assets are also acquired before use. Workers decode local files, with no networking
|
|
145
|
+
in the per-frame path. A fully cached pinned build can render with networking disabled. Build
|
|
146
|
+
snapshots retain immutable prepared-file bindings when a later version changes its assets.
|
|
147
|
+
|
|
148
|
+
A render decodes every video from its **original file**: a build holds only a video's facts, a
|
|
149
|
+
poster and a preview-quality proxy (what the studio plays). Before a frame is drawn, `renderVideo`
|
|
150
|
+
and `renderStills` check that each video's file is there and fail with `AssetFailure` naming the
|
|
151
|
+
ones that aren't; a frame ffmpeg can't decode, or a file shorter than its import counted, fails the
|
|
152
|
+
render with `Frame <n>: Video "<name>": …` and ffmpeg's reason. A poster never stands in for
|
|
153
|
+
footage in a render (that is the player's fallback while frames arrive).
|
|
154
|
+
|
|
155
|
+
A local video is decoded from its source path under the film's root, unless `sources` says otherwise:
|
|
156
|
+
|
|
157
|
+
```ts
|
|
158
|
+
const film = yield* openFilm("renders/1234/dist", { // a build, without the project around it
|
|
159
|
+
sources: { screen: "/data/media/screen.mp4", camera: "/data/media/camera.mp4" },
|
|
160
|
+
});
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
A build inside a project (`my-film/dist`) finds local footage in the project; a build on its own
|
|
164
|
+
needs `sources` for local videos that aren't beside it. Remote originals are resolved automatically
|
|
165
|
+
from the pinned registry in the manifest.
|
|
166
|
+
|
|
167
|
+
## Sound processing (extensions)
|
|
168
|
+
|
|
169
|
+
A render can put part of its soundtrack through processing of your own (voice enhancement, noise
|
|
170
|
+
removal, levelling, effects) and still produce one MP4, in sync. The SDK defines the contract and
|
|
171
|
+
runs it; what a processor does and what it needs (a model, a native tool, a service) is yours, so
|
|
172
|
+
nobody who doesn't use it installs it.
|
|
173
|
+
|
|
174
|
+
```ts
|
|
175
|
+
import { AudioProcessor, openFilm, renderVideo, Sound } from "@tongue-core/sdk";
|
|
176
|
+
|
|
177
|
+
const enhance = yield* enhanceVoice({ peakDb: -3 }); // an AudioProcessor (below)
|
|
178
|
+
yield* renderVideo(film, {
|
|
179
|
+
sound: [
|
|
180
|
+
Sound.track("voiceover", enhance), // the clips of the track with id "voiceover"
|
|
181
|
+
Sound.track("voiceover", level), // the same selection again: runs after enhance
|
|
182
|
+
Sound.master(loudness), // then the whole mix
|
|
183
|
+
],
|
|
184
|
+
});
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
**What a rule takes.** `Sound.track(id, p)`; `Sound.stem({ clips: ["a", "b"] }, p)`;
|
|
188
|
+
`Sound.stem({ voice: true }, p)` (every narration clip); `Sound.master(p)` (the whole mix, after
|
|
189
|
+
the stems, before the limiter). A selection is data (`Sound.AudioSelection` decodes it from flags or
|
|
190
|
+
JSON). The selected clips are mixed into a **stem** of their own (their trims, speed, gain and fades
|
|
191
|
+
applied) across the whole film, processed, and added back; everything else sounds as it always has.
|
|
192
|
+
A track or clip the film doesn't have, or a visual track, is an `InvalidRequest` listing what it
|
|
193
|
+
does have; a clip goes into one stem, so overlapping selections are refused unless they are the same
|
|
194
|
+
(then their processors run in order).
|
|
195
|
+
|
|
196
|
+
**The contract.** A processor gets a `Stem`: `samples` per channel (the whole film at 48 kHz, from
|
|
197
|
+
its first frame), `channels` (2), the stem as a float WAV `file` and as decoded `pcm`, the `clips` in
|
|
198
|
+
it and the `plan`, a `workDir`, and `progress(0..1)`. It returns `{ pcm }` or `{ file }`, plus an
|
|
199
|
+
optional `report` (any JSON). What it returns must have exactly the stem's rate, channels and
|
|
200
|
+
length, and only finite samples, or the render fails with `ProcessorFailure`: that is what keeps
|
|
201
|
+
the sound in sync, so a processor with a delay (a model's lookahead) takes it out itself. Any
|
|
202
|
+
failure of the processor, or a defect in it, is a `ProcessorFailure` naming it.
|
|
203
|
+
|
|
204
|
+
**Two ways to make one.** `AudioProcessor.fromPcm({ key, process: (pcm, stem) => pcm })` for
|
|
205
|
+
processing that needs only the samples. `AudioProcessor.make({ key, process: (stem) => Effect })`
|
|
206
|
+
for anything else: the services `process` uses are taken from where `make` runs and travel with the
|
|
207
|
+
processor, so the render itself never needs them:
|
|
208
|
+
|
|
209
|
+
```ts
|
|
210
|
+
// voice-enhance: its own package, depending on @tongue-core/sdk and on what it needs.
|
|
211
|
+
interface DenoiserOperations {
|
|
212
|
+
readonly name: string;
|
|
213
|
+
readonly denoise: (channel: Float32Array, sampleRate: number) => Effect.Effect<Float32Array>;
|
|
214
|
+
}
|
|
215
|
+
export class Denoiser extends Context.Service<Denoiser, DenoiserOperations>()("voice-enhance/Denoiser") {
|
|
216
|
+
static readonly layer = Layer.effect(Denoiser, loadModel); // the heavy part, paid once, only by those who provide it
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
export const enhanceVoice = (settings: EnhanceSettings) =>
|
|
220
|
+
AudioProcessor.make({
|
|
221
|
+
key: `voice-enhance/v1 peak ${settings.peakDb} dB`,
|
|
222
|
+
process: (stem) => Effect.gen(function* () {
|
|
223
|
+
const denoiser = yield* Denoiser;
|
|
224
|
+
const pcm = yield* stem.pcm;
|
|
225
|
+
const channels = yield* Effect.forEach(pcm.channels, (c) => denoiser.denoise(c, pcm.sampleRate));
|
|
226
|
+
yield* stem.progress(1);
|
|
227
|
+
return { pcm: { sampleRate: pcm.sampleRate, channels }, report: { denoiser: denoiser.name } };
|
|
228
|
+
}),
|
|
229
|
+
}); // Effect<AudioProcessor, never, Denoiser>
|
|
230
|
+
|
|
231
|
+
// In your command line: make the processor where its Layer is provided.
|
|
232
|
+
const render = Command.make("render", { project: Argument.String("project"), enhance: Flag.String("enhance").pipe(Flag.atLeast(0)) },
|
|
233
|
+
Effect.fn(function* (flags) {
|
|
234
|
+
const film = yield* openFilm(flags.project);
|
|
235
|
+
const enhance = yield* enhanceVoice({ peakDb: -3 });
|
|
236
|
+
return yield* renderVideo(film, { sound: flags.enhance.map((track) => Sound.track(track, enhance)) });
|
|
237
|
+
}),
|
|
238
|
+
).pipe(Command.provide(Denoiser.layer));
|
|
239
|
+
```
|
|
240
|
+
|
|
241
|
+
**The key and the cache.** A processor's `key` names it, its version and its settings. A result is
|
|
242
|
+
cached by the key and the exact audio it was given (`.tongue/cache/sound`, the newest 32 kept): render
|
|
243
|
+
again after changing anything that doesn't change that audio (the picture, another track, the
|
|
244
|
+
range, the scale) and the processor doesn't run; the result says `cached: true`. Change the key
|
|
245
|
+
whenever the output would change.
|
|
246
|
+
|
|
247
|
+
**What happened.** `VideoResult.sound` lists each processor run: the stem, its clips, the
|
|
248
|
+
processor's key, whether it was cached, its seconds and its report. `renderVideoEvents` streams
|
|
249
|
+
`Sound` events (`started`, `progress`, `done`, `cached`) while it runs.
|
|
250
|
+
|
|
251
|
+
**Where it runs.** Processors run in your process, alongside the render workers, while the frames
|
|
252
|
+
are drawn (the soundtrack always has); keep heavy work off its main thread (`effect/workers`, a
|
|
253
|
+
child process), or it slows the hand-off of frames to the workers.
|
|
254
|
+
|
|
255
|
+
## How a video renders
|
|
256
|
+
|
|
257
|
+
```
|
|
258
|
+
coordinator ──segments──▶ worker 1: frames ─▶ raster ─▶ encoder ─▶ piece-000000.mp4
|
|
259
|
+
◀─progress── worker 2 … piece-000280.mp4
|
|
260
|
+
soundtrack: sounds + audio clips ─▶ mix ─▶ AAC audio.m4a
|
|
261
|
+
with sound processing: stems ─▶ processors (here, cached) ─▶ sum ─▶ master ─▶ limiter ─▶ AAC
|
|
262
|
+
join: concat + audio ─(stream copy)─▶ film.mp4, frame count probed
|
|
263
|
+
```
|
|
264
|
+
|
|
265
|
+
- **Workers** are processes (`src/render/worker.ts`), one per physical core by default, fewer when
|
|
266
|
+
the memory available wouldn't hold that many (`src/render/budget.ts`). Each loads
|
|
267
|
+
the film and its raster once and draws every segment it is sent, so the JIT stays warm and glyphs
|
|
268
|
+
and images stay uploaded.
|
|
269
|
+
- **Segments** are a quarter to half a second of frames: what a worker is given at a time. Each
|
|
270
|
+
worker starts on its own contiguous run of them; one that finishes early takes over the later
|
|
271
|
+
half of the longest run left (`src/render/runs.ts`). So every worker draws long contiguous
|
|
272
|
+
stretches, and its video decoders read straight through instead of seeking.
|
|
273
|
+
- **Pieces** are what is encoded: each contiguous stretch a worker draws goes to one encoder, as one
|
|
274
|
+
closed-GOP file; the join copies streams without re-encoding. Which pieces a render makes depends
|
|
275
|
+
on how the work was shared out; the frames don't.
|
|
276
|
+
- **Video clips** are decoded by ffmpeg from their original files, in order, frame after frame, a
|
|
277
|
+
few frames ahead, through a FIFO that holds ffmpeg back when it gets ahead; restarted on jumps,
|
|
278
|
+
a window at a time for clips playing backwards, and closed when unused. A frame that can't be
|
|
279
|
+
decoded fails the render ([Footage](#footage)).
|
|
280
|
+
- **The soundtrack** is mixed in its own process while the workers draw (`soundtrack-process.ts`);
|
|
281
|
+
with sound processing, it writes the stems, the processors run in the coordinator, and it adds
|
|
282
|
+
them back, limits and encodes (`sound/soundtrack.ts`).
|
|
283
|
+
|
|
284
|
+
Rasters (who draws) and encoders (who encodes) are chosen by name, because the choice is made here
|
|
285
|
+
but runs inside the workers:
|
|
286
|
+
|
|
287
|
+
| Raster | |
|
|
288
|
+
|---|---|
|
|
289
|
+
| `native` | the Rust raster built for x86-64-v3 (`@tongue-core/raster-linux-x64`); byte-identical to wasm, 2–3× faster |
|
|
290
|
+
| `wasm` | the engine's embedded WebAssembly raster, the browser player's; runs anywhere |
|
|
291
|
+
| `gpu` | EGL + GLES 3.1; within a few levels of the CPU rasters, never chosen by `auto` |
|
|
292
|
+
|
|
293
|
+
Encoders: `x264` (veryfast, crf 14), `vaapi-hevc` / `vaapi-h264` (constant QP 18 / 16, with
|
|
294
|
+
CPU-conversion variants), and `nvenc-*` / `qsv-*` (written from vendor documentation, untested).
|
|
295
|
+
`auto` takes the first that works of nvenc-hevc, vaapi-hevc, vaapi-hevc-cpuconv, qsv-hevc, x264.
|
|
296
|
+
|
|
297
|
+
On this machine (16 threads, AMD GPU) the 35.5 s demo renders at 1080p with sound in about 12 s
|
|
298
|
+
(≈ 87 fps) with `native` + `vaapi-hevc`.
|
|
299
|
+
|
|
300
|
+
## Errors
|
|
301
|
+
|
|
302
|
+
Every failure is a `Schema.TaggedError` with a message that says what to change. `exitCodeOf`
|
|
303
|
+
gives the CLI's exit code.
|
|
304
|
+
|
|
305
|
+
| Error | When | Exit |
|
|
306
|
+
|---|---|---|
|
|
307
|
+
| `FilmFailure` | the module is missing, throws, isn't a `<Film>`, or its timeline doesn't plan | 2 |
|
|
308
|
+
| `AssetFailure` | an asset's file is missing or unreadable, or a video's original file isn't there to render from | 2 |
|
|
309
|
+
| `InvalidRequest` | a frame outside the film, a bad range or scale | 2 |
|
|
310
|
+
| `ToolFailure` | ffmpeg or ffprobe is missing or failed | 3 |
|
|
311
|
+
| `BackendUnavailable` | the raster or encoder asked for doesn't work here | 3 |
|
|
312
|
+
| `RenderFailure` | a worker died, a frame failed, or a video frame couldn't be decoded | 3 |
|
|
313
|
+
| `ProcessorFailure` | a sound processor failed, or returned audio that would break sync (rate, channels, length) or isn't audio | 3 |
|
|
314
|
+
| `LayoutError`, `CompileError`, `LoadError`, `TimelineError`, `DocError`, `TemplateError` | a film project doesn't build (from `@tongue-core/project`: each names its file, line or path) | 2 |
|
|
315
|
+
|
|
316
|
+
## Environment
|
|
317
|
+
|
|
318
|
+
| Variable | |
|
|
319
|
+
|---|---|
|
|
320
|
+
| `TONGUE_FFMPEG`, `TONGUE_FFPROBE` | the tools' paths (default: `ffmpeg`, `ffprobe` on the PATH) |
|
|
321
|
+
| `TONGUE_NATIVE_RASTER`, `TONGUE_GPU_RASTER` | a raster library to use instead of `@tongue-core/raster-linux-x64`'s |
|
|
322
|
+
| `TONGUE_VAAPI_DEVICE` | the render node for VAAPI and Quick Sync (default `/dev/dri/renderD128`) |
|
|
323
|
+
|
|
324
|
+
## Layout
|
|
325
|
+
|
|
326
|
+
```
|
|
327
|
+
src/
|
|
328
|
+
index.ts, client.ts the Effect API and the Promise facade
|
|
329
|
+
project.ts projects, openFilm and buildFilm
|
|
330
|
+
build/build.ts building a film project on this machine: its files, a module host, the build folder, editor types
|
|
331
|
+
init.ts init from templates
|
|
332
|
+
check.ts, backends.ts checkFilm, listBackends
|
|
333
|
+
import/ asset import: fonts, images, audio, video (ffprobe, posters)
|
|
334
|
+
render/ stills, the video pipeline and its events, its worker and soundtrack processes, video decoding and inputs
|
|
335
|
+
sound/ sound processing: Sound (rules), AudioProcessor (the contract), the soundtrack around them
|
|
336
|
+
points.ts points and ranges in a film, as people write them
|
|
337
|
+
TongueServices.ts the layer the SDK runs on
|
|
338
|
+
encode/encoders.ts the ffmpeg encoders and how each is tried
|
|
339
|
+
raster/ the native and GPU raster bindings (index.ts) and the registry (rasters.ts)
|
|
340
|
+
studio/server.ts the studio's server: the app bundled, a project's build served and rebuilt
|
|
341
|
+
test/ unit tests, end-to-end ones on fixtures/tiny (a film with text, video and sound; sound
|
|
342
|
+
processing and footage that can't be decoded on copies of it), builds rendered with `sources`, and
|
|
343
|
+
projects from every template (project.test.ts: film.json draws the same frames as film.tsx)
|
|
344
|
+
```
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { Effect } from "effect";
|
|
2
|
+
import { type EncoderName } from "./encode/encoders.js";
|
|
3
|
+
import { type RasterName } from "./raster/rasters.js";
|
|
4
|
+
export interface BackendsReport {
|
|
5
|
+
readonly rasters: ReadonlyArray<{
|
|
6
|
+
readonly name: RasterName;
|
|
7
|
+
readonly available: boolean;
|
|
8
|
+
readonly exact: boolean;
|
|
9
|
+
readonly summary: string;
|
|
10
|
+
}>;
|
|
11
|
+
readonly encoders: ReadonlyArray<{
|
|
12
|
+
readonly name: EncoderName;
|
|
13
|
+
readonly available: boolean;
|
|
14
|
+
readonly hardware: boolean;
|
|
15
|
+
readonly tested: boolean;
|
|
16
|
+
readonly summary: string;
|
|
17
|
+
}>;
|
|
18
|
+
/** What `auto` picks here. */
|
|
19
|
+
readonly auto: {
|
|
20
|
+
readonly raster: RasterName;
|
|
21
|
+
readonly encoder: EncoderName | null;
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
export declare const listBackends: () => Effect.Effect<BackendsReport, import("effect/Config").ConfigError, never>;
|
package/dist/backends.js
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { Effect } from "effect";
|
|
2
|
+
import { AUTO_ENCODERS, DEFAULT_SETTINGS, ENCODER_NAMES, ENCODER_SUMMARIES, encoderSpec, encoderWorks } from "./encode/encoders.js";
|
|
3
|
+
import { autoRaster, RASTERS } from "./raster/rasters.js";
|
|
4
|
+
import { ffmpegPath } from "./tools.js";
|
|
5
|
+
export const listBackends = Effect.fn("listBackends")(function* () {
|
|
6
|
+
const ffmpeg = yield* ffmpegPath;
|
|
7
|
+
const encoders = yield* Effect.forEach(ENCODER_NAMES, (name) => Effect.promise(async () => {
|
|
8
|
+
const spec = encoderSpec(name, DEFAULT_SETTINGS, 30);
|
|
9
|
+
return { name, available: await encoderWorks(ffmpeg, spec), hardware: spec.hardware, tested: spec.tested, summary: ENCODER_SUMMARIES[name] };
|
|
10
|
+
}), { concurrency: "unbounded" });
|
|
11
|
+
const report = {
|
|
12
|
+
rasters: RASTERS.map((r) => ({ name: r.name, available: r.available(), exact: r.exact, summary: r.summary })),
|
|
13
|
+
encoders,
|
|
14
|
+
auto: { raster: autoRaster().name, encoder: AUTO_ENCODERS.find((n) => encoders.some((e) => e.name === n && e.available)) ?? null },
|
|
15
|
+
};
|
|
16
|
+
return report;
|
|
17
|
+
});
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import type { Asset, MediaInfo, SceneDefinition } from "@tongue-core/film";
|
|
2
|
+
import { type Build, type ModuleHost, type ProjectFiles } from "@tongue-core/project";
|
|
3
|
+
import { Effect, FileSystem, type PlatformError } from "effect";
|
|
4
|
+
import { type ImportReport } from "../import/importer.js";
|
|
5
|
+
import type { Project } from "../project.js";
|
|
6
|
+
import { type MediaOptions } from "../media/cache.js";
|
|
7
|
+
/** A project's code and data: every .ts, .tsx and .json file outside hidden folders and the ones
|
|
8
|
+
* tools write. */
|
|
9
|
+
export declare const readProjectFiles: (root: string) => Effect.Effect<ProjectFiles, PlatformError.PlatformError, FileSystem.FileSystem>;
|
|
10
|
+
/** Loads bundles by writing each to `dir` (named by its hash, so written once) and importing it. */
|
|
11
|
+
export declare const fileModuleHost: (dir: string) => ModuleHost;
|
|
12
|
+
/** Points the project's `.tongue/types/film` at this SDK's @tongue-core/film (tsconfig.json reads it). */
|
|
13
|
+
export declare const linkFilmTypes: (root: string) => Effect.Effect<void, PlatformError.PlatformError, FileSystem.FileSystem>;
|
|
14
|
+
export interface BuiltProject {
|
|
15
|
+
readonly assetsDir: string;
|
|
16
|
+
readonly build: Build;
|
|
17
|
+
/** Where the build was written. */
|
|
18
|
+
readonly dir: string;
|
|
19
|
+
readonly imported: ImportReport;
|
|
20
|
+
/** The scenes the bundles export, by id: what film.json is assembled with. */
|
|
21
|
+
readonly scenes: ReadonlyMap<string, SceneDefinition>;
|
|
22
|
+
readonly media: MediaInfo;
|
|
23
|
+
}
|
|
24
|
+
/** The imported files a film's assets are (a video is its facts, its poster and its proxy's
|
|
25
|
+
* segments), with their sizes and hashes: what build.json lists. */
|
|
26
|
+
export declare const importedFiles: (project: Project, assets: readonly Asset[]) => Effect.Effect<{
|
|
27
|
+
readonly path: string;
|
|
28
|
+
readonly bytes: number;
|
|
29
|
+
readonly hash: string;
|
|
30
|
+
readonly url?: string | undefined;
|
|
31
|
+
}[], PlatformError.PlatformError, FileSystem.FileSystem>;
|
|
32
|
+
/** How a written build reaches its assets: copies of the files (a build to publish), or a link to
|
|
33
|
+
* the project's imported assets (a build for Tongue's own use). */
|
|
34
|
+
export type AssetPlacement = {
|
|
35
|
+
readonly kind: "copy";
|
|
36
|
+
readonly from: string;
|
|
37
|
+
} | {
|
|
38
|
+
readonly kind: "link";
|
|
39
|
+
readonly to: string;
|
|
40
|
+
};
|
|
41
|
+
/** Writes a build to `dir`, replacing the build that was there (old bundles and assets included). */
|
|
42
|
+
export declare const writeBuildFiles: (dir: string, build: Build, assets: AssetPlacement) => Effect.Effect<void, PlatformError.PlatformError, FileSystem.FileSystem>;
|
|
43
|
+
/** Keeps a build in `.tongue/builds/<id>`, named by its content and never rewritten, so renders
|
|
44
|
+
* reading one build are never disturbed by the next. Written to a temporary folder first and moved
|
|
45
|
+
* into place, so a build folder is either whole or not there. */
|
|
46
|
+
export declare const keepBuild: (project: Project, build: Build) => Effect.Effect<string, PlatformError.PlatformError, FileSystem.FileSystem>;
|
|
47
|
+
/** Builds a film project into `dir` (what `tongue build` does: dist/ by default), replacing what
|
|
48
|
+
* was there, with copies of its imported assets: a folder any static host can serve. */
|
|
49
|
+
export declare const buildFilmProject: (project: Project, dir: string, options?: MediaOptions | undefined) => Effect.Effect<BuiltProject, import("../errors.js").AssetFailure | import("../errors.js").ToolFailure | import("@tongue-core/project/media").MediaFailure | import("@tongue-core/project").LayoutError | import("@tongue-core/project").CompileError | import("@tongue-core/project").LoadError | import("@tongue-core/project").TimelineError | import("@tongue-core/project").DocError | import("effect/Config").ConfigError | PlatformError.PlatformError | import("effect/Schema").SchemaError, import("effect/process/ChildProcessSpawner").ChildProcessSpawner | FileSystem.FileSystem | import("effect/Path").Path>;
|
|
50
|
+
/** Builds a film project for Tongue's own use (stills, renders, dev): kept in .tongue/builds. */
|
|
51
|
+
export declare const buildForWork: (project: Project, options?: MediaOptions | undefined) => Effect.Effect<BuiltProject, import("../errors.js").AssetFailure | import("../errors.js").ToolFailure | import("@tongue-core/project/media").MediaFailure | import("@tongue-core/project").LayoutError | import("@tongue-core/project").CompileError | import("@tongue-core/project").LoadError | import("@tongue-core/project").TimelineError | import("@tongue-core/project").DocError | import("effect/Config").ConfigError | PlatformError.PlatformError | import("effect/Schema").SchemaError, import("effect/process/ChildProcessSpawner").ChildProcessSpawner | FileSystem.FileSystem | import("effect/Path").Path>;
|