@minimoji/heads 0.0.0-stage → 19.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE-SDK +41 -0
- package/THIRD_PARTY_NOTICES.md +44 -0
- package/emote-bursts/anger.png +0 -0
- package/emote-bursts/bang.png +0 -0
- package/emote-bursts/bolt.png +0 -0
- package/emote-bursts/cloud.png +0 -0
- package/emote-bursts/drop.png +0 -0
- package/emote-bursts/flame.png +0 -0
- package/emote-bursts/ha.png +0 -0
- package/emote-bursts/heart.png +0 -0
- package/emote-bursts/puff.png +0 -0
- package/emote-bursts/sparkle.png +0 -0
- package/emote-bursts/star.png +0 -0
- package/emote-bursts/thought-empty.png +0 -0
- package/emote-bursts/thought.png +0 -0
- package/emote-bursts/z.png +0 -0
- package/engine/bubble-captions.d.ts +17 -0
- package/engine/clip.d.ts +86 -0
- package/engine/line.d.ts +172 -0
- package/minimoji-heads-bursts.js +2 -0
- package/minimoji-heads.d.ts +307 -0
- package/minimoji-heads.js +3736 -0
- package/package.json +31 -4
- package/parts/beret.bin +0 -0
- package/parts/bucket.bin +0 -0
- package/parts/cap.bin +0 -0
- package/parts/flatCap.bin +0 -0
- package/parts/sculptedAfro.bin +0 -0
- package/parts/sculptedBob.bin +0 -0
- package/parts/sculptedCrop.bin +0 -0
- package/parts/sculptedCurls.bin +0 -0
- package/parts/sculptedHighBun.bin +0 -0
- package/parts/sculptedPigtails.bin +0 -0
- package/parts/sculptedQuiff.bin +0 -0
- package/parts/sculptedSidePart.bin +0 -0
- package/parts/sculptedSpaceBuns.bin +0 -0
- package/parts/sculptedTopKnot.bin +0 -0
- package/parts/sculptedWaves.bin +0 -0
- package/README.md +0 -3
package/LICENSE-SDK
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
Minimoji SDK License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Hypersocial Inc. All rights reserved.
|
|
4
|
+
|
|
5
|
+
This licence covers the Minimoji Swift package (Package.swift and Sources/)
|
|
6
|
+
and the npm packages @minimoji/react and @minimoji/heads (together, "the
|
|
7
|
+
SDK"). The CLI, the agent skills and the documentation in this repository
|
|
8
|
+
are MIT-licensed instead (LICENSE).
|
|
9
|
+
|
|
10
|
+
1. Use. You may use the SDK to build applications and websites, and ship,
|
|
11
|
+
sell or charge for those applications, including the SDK inside them.
|
|
12
|
+
|
|
13
|
+
2. Redistribution. You may redistribute the SDK only as part of such an
|
|
14
|
+
application or website, not on its own or as part of another SDK,
|
|
15
|
+
library or developer tool.
|
|
16
|
+
|
|
17
|
+
3. No extraction. You may not reverse engineer, decompile, de-minify or
|
|
18
|
+
otherwise attempt to recover source code from the SDK, or extract the
|
|
19
|
+
engine bundles it contains for use outside the SDK, except to the extent
|
|
20
|
+
applicable law expressly permits it despite this restriction.
|
|
21
|
+
|
|
22
|
+
4. No competing products. You may not use the SDK, or anything learned from
|
|
23
|
+
it, to build an avatar engine, avatar SDK or avatar service that competes
|
|
24
|
+
with Minimoji.
|
|
25
|
+
|
|
26
|
+
5. The service. Avatars, characters and other content come from the Minimoji
|
|
27
|
+
service under its own terms, through keys issued to you. This licence
|
|
28
|
+
grants no right to the service.
|
|
29
|
+
|
|
30
|
+
6. Third-party software. The SDK includes third-party software under its own
|
|
31
|
+
licences, listed in THIRD_PARTY_NOTICES.md. Those licences govern those
|
|
32
|
+
parts.
|
|
33
|
+
|
|
34
|
+
7. No warranty. THE SDK IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
|
35
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
|
36
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN
|
|
37
|
+
NO EVENT SHALL HYPERSOCIAL INC. BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
38
|
+
LIABILITY ARISING FROM, OUT OF OR IN CONNECTION WITH THE SDK OR ITS USE.
|
|
39
|
+
|
|
40
|
+
8. Termination. This licence ends automatically if you breach it. On
|
|
41
|
+
termination you must stop distributing the SDK.
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# Third-party notices
|
|
2
|
+
|
|
3
|
+
Minimoji includes or adapts the following third-party software.
|
|
4
|
+
|
|
5
|
+
## lipsync-engine
|
|
6
|
+
|
|
7
|
+
The live viseme analysis (`gvisemes.js` in the engine, bundled into
|
|
8
|
+
`minimoji-engine.js` and the web bundles; `MinimojiLive/LiveVisemes.swift`)
|
|
9
|
+
is adapted from lipsync-engine's `FrequencyAnalyzer.js` and
|
|
10
|
+
`extractBandEnergies`.
|
|
11
|
+
|
|
12
|
+
```
|
|
13
|
+
MIT License
|
|
14
|
+
|
|
15
|
+
Copyright (c) 2025 Beer Digital LLC
|
|
16
|
+
|
|
17
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
18
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
19
|
+
in the Software without restriction, including without limitation the rights
|
|
20
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
21
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
22
|
+
furnished to do so, subject to the following conditions:
|
|
23
|
+
|
|
24
|
+
The above copyright notice and this permission notice shall be included in all
|
|
25
|
+
copies or substantial portions of the Software.
|
|
26
|
+
|
|
27
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
28
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
29
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
30
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
31
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
32
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
33
|
+
SOFTWARE.
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
## three.js
|
|
37
|
+
|
|
38
|
+
MIT License, Copyright © 2010-2026 three.js authors. Bundled into
|
|
39
|
+
`minimoji-engine.js` and `minimoji-heads.js`; a peer dependency of
|
|
40
|
+
`@minimoji/react`. https://github.com/mrdoob/three.js/blob/dev/LICENSE
|
|
41
|
+
|
|
42
|
+
## KayKit animations
|
|
43
|
+
|
|
44
|
+
CC0 1.0. Shipped in `MinimojiBody/Resources` with their own licence files.
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { LineWord, SpeechLine } from './line';
|
|
2
|
+
|
|
3
|
+
/** One bubble caption: a whole phrase (usually a sentence, split near BUBBLE_CAPTION_CHARS at punctuation),
|
|
4
|
+
* shown alone in a speech bubble beside the head. Not the 4-word phrases of `captionWindow` /
|
|
5
|
+
* <MinimojiCaptions>, which suit a caption strip: bubble captions are sentence-sized, for a speech bubble. */
|
|
6
|
+
export interface BubbleCaption { start: number; end: number; words: LineWord[] }
|
|
7
|
+
/** the longest caption a bubble shows at once (64 characters, about two bubble lines) */
|
|
8
|
+
export const BUBBLE_CAPTION_CHARS: number;
|
|
9
|
+
/** seconds a word's pill stays lit after the word ends (0.12) */
|
|
10
|
+
export const BUBBLE_LIT_HOLD: number;
|
|
11
|
+
/** a line's bubble captions: its own phrases one at a time, a long one split at punctuation, never a lone word.
|
|
12
|
+
* Without timings, `text` is one caption with words 0.3 s apart (none for an empty text) */
|
|
13
|
+
export function capsFromLine(line: Pick<SpeechLine, 'captions' | 'words'> | null | undefined, text?: string, opts?: { maxChars?: number }): BubbleCaption[];
|
|
14
|
+
/** the caption showing at `t` seconds (the last one holds after the line ends) and its lit word (-1 between words) */
|
|
15
|
+
export function captionAt(caps: readonly BubbleCaption[], t: number): { index: number; lit: number };
|
|
16
|
+
/** when `word` is first said (seconds), ignoring case and punctuation; `fallback` when it isn't */
|
|
17
|
+
export function wordTime(line: Pick<SpeechLine, 'words'> | null | undefined, word: string, fallback: number): number;
|
package/engine/clip.d.ts
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
// Types for @minimoji/engine/clip — a plain audio clip → a line (fetch, decode, buildLine, cache).
|
|
2
|
+
import type { AuthoredCue, LineAudio, LineCaption, LineLetter, LineWord, ProviderAlignment, SpeechLine } from './line.js';
|
|
3
|
+
|
|
4
|
+
/** a plain clip: the audio, and whatever the host knows about its words */
|
|
5
|
+
export interface SpeechClip {
|
|
6
|
+
/** a URL (fetched), a Blob or bytes (decoded with Web Audio), or an AudioBuffer (its own samples) */
|
|
7
|
+
audio: string | Blob | ArrayBuffer | ArrayBufferView | AudioBuffer;
|
|
8
|
+
/** the line's text: one caption across the voiced part (no timed words) */
|
|
9
|
+
transcript?: string | null;
|
|
10
|
+
/** timed words (seconds): word captions, sentence captions and the word cues (a nod on "yes") */
|
|
11
|
+
words?: LineWord[] | null;
|
|
12
|
+
/** timed letters (seconds, spaces included): the mouth shaped from the words, lips meeting on m/b/p only. Unset with
|
|
13
|
+
* `words` (and no `alignment`): the words spread into letters (`lettersFromWords`). `false` (or `[]`): no letters,
|
|
14
|
+
* the mouth guessed from the sound */
|
|
15
|
+
letters?: LineLetter[] | false | null;
|
|
16
|
+
/** or the TTS provider's character alignment as it came (ElevenLabs' `alignment`): read into `letters` */
|
|
17
|
+
alignment?: ProviderAlignment | ProviderAlignment[] | null;
|
|
18
|
+
/** a string: the whole line's caption when there are no words or transcript; an array: the captions, as is (one that
|
|
19
|
+
* ends past the audio ends with it; one that starts past it is dropped) */
|
|
20
|
+
captions?: string | LineCaption[] | null;
|
|
21
|
+
/** the face the line starts on */
|
|
22
|
+
mood?: string | null;
|
|
23
|
+
/** the app's own direction, merged with the derived cues: `{ at, gesture | mood | brow | look }` with `at` in seconds
|
|
24
|
+
* or on a word (`{ word: 7 }`, `{ word: 'card', nth? }`); a word the line does not say fails the clip */
|
|
25
|
+
cues?: AuthoredCue[] | null;
|
|
26
|
+
/** a recording with no words: `true` (or `{ key, service }`, or unset when the cache holds a transcriber and the clip
|
|
27
|
+
* has no words, transcript or captions) sends the decoded audio to Minimoji's POST /lines, whose timed words give the
|
|
28
|
+
* line its captions and word cues; `false` never sends it. An asked-for transcription that fails fails the clip; one
|
|
29
|
+
* chosen because the clip had no words falls back to the plain line. The audio goes to Minimoji and ElevenLabs, not stored */
|
|
30
|
+
transcribe?: boolean | ClipTranscriber | null;
|
|
31
|
+
}
|
|
32
|
+
/** where a recording is transcribed: the app's publishable key (`mk_pub_…`) and the service's `.convex.site` origin */
|
|
33
|
+
export interface ClipTranscriber { key: string; service: string }
|
|
34
|
+
/** one timed letter inside a word (seconds) */
|
|
35
|
+
|
|
36
|
+
export const CLIP_SAMPLE_RATE: number;
|
|
37
|
+
export const CLIP_CACHE_SIZE: number;
|
|
38
|
+
export const CLIP_TIMEOUT: number;
|
|
39
|
+
export const TRANSCRIBE_SAMPLE_RATE: number;
|
|
40
|
+
export const TRANSCRIBE_MAX_SECONDS: number;
|
|
41
|
+
export const TRANSCRIBE_TIMEOUT: number;
|
|
42
|
+
/** samples → a 16-bit mono WAV at `rate` (16 kHz) */
|
|
43
|
+
export function encodeWav(pcm: Float32Array, sampleRate: number, rate?: number): Uint8Array;
|
|
44
|
+
/** samples → the words POST /lines heard in them; throws with the service's words on a refusal */
|
|
45
|
+
/** with `text`, the service aligns that script to the audio (`alignedTo: 'text'`, `loss`) instead of transcribing it */
|
|
46
|
+
export function transcribeSamples(audio: { pcm: Float32Array; sampleRate: number }, opts: ClipTranscriber & { text?: string | null; fetch?: (url: string, init: RequestInit) => Promise<Response>; signal?: AbortSignal }): Promise<{ words: LineWord[]; letters: LineLetter[]; transcript: string; alignedTo: 'text' | 'transcript'; loss?: number }>;
|
|
47
|
+
/** the clip as a cache holding `transcriber` prepares it */
|
|
48
|
+
export function withTranscriber(clip: SpeechClip, transcriber: ClipTranscriber | null | undefined): SpeechClip;
|
|
49
|
+
export function isClip(x: unknown): x is SpeechClip;
|
|
50
|
+
/** a line spoken from text by POST /lines/say: no audio of its own */
|
|
51
|
+
export interface TextClip { text: string; voice: string | { voiceId: string; effect?: 'none' | 'robot'; settings?: Record<string, number | boolean> }; mood?: string | null; model?: string;
|
|
52
|
+
/** where it is spoken: the app's publishable key and the service origin (or the session's `synthesize`), and optionally the caller's own ElevenLabs key */
|
|
53
|
+
synthesize?: ClipSynthesizer | null }
|
|
54
|
+
export interface ClipSynthesizer { key: string; service: string; elevenLabsKey?: string | null }
|
|
55
|
+
export interface SaidLine { audio: string; format: 'wav'; durationMs: number; line: SpeechLine; words: LineWord[]; letters: LineLetter[]; transcript: string; chars: number; charged: number; cached: boolean; voice: { name: string | null; voiceId: string; effect: string } }
|
|
56
|
+
export function isTextClip(x: unknown): x is TextClip;
|
|
57
|
+
export const SYNTHESIZE_TIMEOUT: number;
|
|
58
|
+
export function synthesizeLine(clip: TextClip, opts: ClipSynthesizer & { fetch?: (url: string, init: RequestInit) => Promise<Response>; signal?: AbortSignal }): Promise<SaidLine>;
|
|
59
|
+
export function monoPCM(buffer: AudioBuffer): Float32Array;
|
|
60
|
+
export function decodeAudio(bytes: ArrayBuffer | ArrayBufferView, sampleRate?: number): Promise<{ pcm: Float32Array; sampleRate: number }>;
|
|
61
|
+
export interface ClipCache {
|
|
62
|
+
/** fetch, decode and build a clip (the same promise for one already on its way); rejects on a load or decode failure */
|
|
63
|
+
prepare(clip: SpeechClip): Promise<SpeechLine>;
|
|
64
|
+
/** the built line when the clip is cached, synchronously; null otherwise */
|
|
65
|
+
peek(clip: SpeechClip): SpeechLine | null;
|
|
66
|
+
readonly size: number;
|
|
67
|
+
clear(): void;
|
|
68
|
+
}
|
|
69
|
+
export function createClipCache(opts?: {
|
|
70
|
+
size?: number;
|
|
71
|
+
sampleRate?: number;
|
|
72
|
+
/** the network: a clip's URL, and POST /lines for a transcribed clip */
|
|
73
|
+
fetch?: (url: string, init?: RequestInit) => Promise<{ ok: boolean; status: number; arrayBuffer(): Promise<ArrayBuffer>; json(): Promise<any> }>;
|
|
74
|
+
/** the transcriber a clip with no words uses (and one saying `transcribe: true`) */
|
|
75
|
+
transcribe?: ClipTranscriber | null;
|
|
76
|
+
decode?: (bytes: ArrayBuffer | ArrayBufferView, sampleRate: number) => Promise<{ pcm: Float32Array; sampleRate: number }>;
|
|
77
|
+
/** seconds before a clip that has not loaded and built fails (30) */
|
|
78
|
+
timeout?: number;
|
|
79
|
+
build?: (input: { pcm: Float32Array; sampleRate: number; words: LineWord[] | null; letters: LineLetter[] | null; transcript: string | null; mood: string | null; captions: LineCaption[] | null }) => SpeechLine | Promise<SpeechLine>;
|
|
80
|
+
}): ClipCache;
|
|
81
|
+
export function clipSource(audio: SpeechClip['audio']): { src: string; revoke(): void } | null;
|
|
82
|
+
export function audioBufferMedia(buffer: AudioBuffer, context: AudioContext): LineAudio & { readonly paused: boolean };
|
|
83
|
+
/** the longest text POST /lines aligns */
|
|
84
|
+
export const ALIGN_MAX_TEXT: number;
|
|
85
|
+
/** the text a transcribed clip sends to be aligned (its transcript, if the service can align it), or null */
|
|
86
|
+
export function scriptOf(clip: { transcript?: string | null; words?: unknown[] | null; letters?: unknown[] | null; alignment?: unknown }): string | null;
|
package/engine/line.d.ts
ADDED
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
// Types for @minimoji/engine/line — the talking-line format (docs/speech-line-format.md) and its builder/player.
|
|
2
|
+
|
|
3
|
+
export type LineFrame = [open: number, width: number, round: number, level: number, pitch?: number | null];
|
|
4
|
+
export interface LineCaption { text: string; start: number; end: number; kind?: string }
|
|
5
|
+
export interface LineWord { text: string; start: number; end: number }
|
|
6
|
+
/** one timed character (seconds) of what was said, spaces included: what shapes a line's mouth from the words */
|
|
7
|
+
export interface LineLetter { text: string; start: number; end: number }
|
|
8
|
+
export type LineCue =
|
|
9
|
+
| { at: number; mood: string; k?: number }
|
|
10
|
+
| { at: number; gesture: string }
|
|
11
|
+
| { at: number; brow: 'up' | 'down'; k: number }
|
|
12
|
+
/** the eyes turned to `{ x, y }` (each -1..1), aside ('away') or back to the camera ('viewer' / null), for `hold` seconds */
|
|
13
|
+
| { at: number; look: LookTarget; hold?: number };
|
|
14
|
+
export type LookTarget = { x: number; y: number } | 'away' | 'viewer' | null;
|
|
15
|
+
/** where an app's cue lands: seconds, or a word of the line (`{ word: index }` from 0, or `{ word: 'text', nth? }`) */
|
|
16
|
+
export type CueAt = number | { word: number } | { word: string; nth?: number };
|
|
17
|
+
/** a cue an app hands `buildLine` (or `say(clip)` as `clip.cues`): any kind, `at` in seconds or on a word; a nod or
|
|
18
|
+
* shake (`{ at, nod: true }`) plays the face's own track */
|
|
19
|
+
export type AuthoredCue =
|
|
20
|
+
| (Omit<Extract<LineCue, { mood: string }>, 'at'> & { at: CueAt })
|
|
21
|
+
| (Omit<Extract<LineCue, { gesture: string }>, 'at'> & { at: CueAt })
|
|
22
|
+
| (Omit<Extract<LineCue, { brow: 'up' | 'down' }>, 'at'> & { at: CueAt })
|
|
23
|
+
| (Omit<Extract<LineCue, { look: LookTarget }>, 'at'> & { at: CueAt })
|
|
24
|
+
| { at: CueAt; nod: true }
|
|
25
|
+
| { at: CueAt; shake: true };
|
|
26
|
+
export interface SpeechLine {
|
|
27
|
+
version: 1;
|
|
28
|
+
fps: 60;
|
|
29
|
+
duration: number;
|
|
30
|
+
frames: LineFrame[];
|
|
31
|
+
captions: LineCaption[];
|
|
32
|
+
words?: LineWord[];
|
|
33
|
+
cues: LineCue[];
|
|
34
|
+
}
|
|
35
|
+
export interface LineSample {
|
|
36
|
+
time: number;
|
|
37
|
+
mouth: { open: number; width: number; round: number; energy: number; bliss: number; pitch?: number };
|
|
38
|
+
level: number;
|
|
39
|
+
caption: string;
|
|
40
|
+
}
|
|
41
|
+
export type CaptionStyle = 'phrases' | 'window';
|
|
42
|
+
export interface CaptionWindow {
|
|
43
|
+
text: string;
|
|
44
|
+
/** `lit`: the word being said (phrases only) */
|
|
45
|
+
words: { text: string; opacity: number; lit?: boolean }[];
|
|
46
|
+
/** the phrase's index in the line (phrases only): a view moves a new one in */
|
|
47
|
+
phrase?: number;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export const LINE_VERSION: 1;
|
|
51
|
+
export const LINE_FPS: 60;
|
|
52
|
+
export const MAX_LINE_SECONDS: number;
|
|
53
|
+
export const MAX_CAPTIONS: number;
|
|
54
|
+
export const MAX_WORDS: number;
|
|
55
|
+
export const MAX_LETTERS: number;
|
|
56
|
+
/** letter (lower case) → the mouth shape it takes, or null (the sound's own shape, never shut); two-letter rules too */
|
|
57
|
+
export const LETTER_SHAPES: Readonly<Record<string, 'rest' | 'open' | 'round' | 'wide' | 'shut' | null>>;
|
|
58
|
+
/** a TTS provider's character alignment (ElevenLabs': one object or an array of chunks) → timed letters, `[tags]` dropped */
|
|
59
|
+
export function lettersFromAlignment(alignment: ProviderAlignment | ProviderAlignment[] | null | undefined): LineLetter[];
|
|
60
|
+
/** timed words only (a transcriber's) → timed letters, each word's spread evenly over its span */
|
|
61
|
+
export function lettersFromWords(words: { text: string; start: number; end: number }[]): LineLetter[];
|
|
62
|
+
export interface ProviderAlignment { characters: string[]; character_start_times_seconds: number[]; character_end_times_seconds: number[] }
|
|
63
|
+
export const CAPTION_TEXT: number;
|
|
64
|
+
export const CAPTION_WORDS: number;
|
|
65
|
+
export const CAPTION_CHARS: number;
|
|
66
|
+
export const CAPTION_FADE: number;
|
|
67
|
+
export const CAPTION_PHRASE_WORDS: number;
|
|
68
|
+
export const CAPTION_SAID: number;
|
|
69
|
+
export const CAPTION_AHEAD: number;
|
|
70
|
+
/** a line's words in phrases, as `[from, to)` index ranges */
|
|
71
|
+
export function linePhrases(words: { text: string }[], opts?: { maxWords?: number; maxChars?: number }): [number, number][];
|
|
72
|
+
export const MAX_CUES: number;
|
|
73
|
+
export const QUIET_K: number;
|
|
74
|
+
/** the intensity of a face the builder sets on its own (a line's `mood`, a tag): under QUIET_K, no emote burst */
|
|
75
|
+
export const SET_K: number;
|
|
76
|
+
/** audio tag (lower case, no brackets) → the face it sets */
|
|
77
|
+
export const TAG_MOODS: Readonly<Record<string, string>>;
|
|
78
|
+
/** directing tag (lower case, no brackets: `nod`, `shake`, `brow`, `brow down`, `look away`, `look back`) → its cue */
|
|
79
|
+
export const TAG_CUES: Readonly<Record<string, object>>;
|
|
80
|
+
/** seconds a look cue holds when it names no `hold`, and the longest it may ask for */
|
|
81
|
+
export const LOOK_HOLD: number;
|
|
82
|
+
export const MAX_LOOK_HOLD: number;
|
|
83
|
+
/** seconds around an authored accent where a word rule's accent is dropped */
|
|
84
|
+
export const GAP: number;
|
|
85
|
+
/** where an 'away' look turns the eyes (x mirrored on alternate looks) */
|
|
86
|
+
export const LOOK_AWAY: Readonly<{ x: number; y: number }>;
|
|
87
|
+
/** timed words with `[tags]` → the words without them, what the word rules read, the faces the tags set and the directing tags' cues */
|
|
88
|
+
export function splitTags<W extends { text: string; start: number; end: number }>(words: W[]): { words: W[]; said: W[]; moods: LineCue[]; directed: AuthoredCue[] };
|
|
89
|
+
/** a directed nod / shake → the gesture the face held then plays for it (knowing-nod / head-shake on a neutral face) */
|
|
90
|
+
export function directedCues(cues: ({ at: number } & Record<string, unknown>)[], arc?: LineCue[]): LineCue[];
|
|
91
|
+
/** a look cue's target → the point for `life.lookAt`, or null for the viewer */
|
|
92
|
+
export function lookPoint(target: LookTarget, side?: 1 | -1): { x: number; y: number } | null;
|
|
93
|
+
/** holds a look cue for a stage that draws frames: `tick(dt)` lets it go after its hold */
|
|
94
|
+
export function createLookHold(lookAt?: (x: number | null, y?: number) => void): {
|
|
95
|
+
look(target: LookTarget, hold?: number): void; tick(dt: number): void; release(): void; readonly current: { x: number; y: number } | null;
|
|
96
|
+
};
|
|
97
|
+
/** the stage interface a cue plays on */
|
|
98
|
+
export interface CueStage {
|
|
99
|
+
setMood(mood: string, id?: unknown): void;
|
|
100
|
+
emote(mood: string, id?: unknown, at?: number): void;
|
|
101
|
+
gesture?(gesture: string, id?: unknown): void;
|
|
102
|
+
brow?(brow: 'up' | 'down', k: number, id?: unknown): void;
|
|
103
|
+
look?(target: { x: number; y: number } | 'away' | null, hold: number, id?: unknown): void;
|
|
104
|
+
}
|
|
105
|
+
/** a stage for `playCues` on one head's life (`head.life`); on a <MinimojiHead> give it `look="manual"` */
|
|
106
|
+
export function cueStage(life: { setFace(mood: string): void; perform(what: string | { mood: string }): unknown; lookAt(x: number | null, y?: number): void }): CueStage & { release(): void };
|
|
107
|
+
/** a cue track with no line, on its own clock (a function answering seconds, ticked by the host; or none: timers).
|
|
108
|
+
* `done` resolves 'ended' with the last cue (a look it turned may still be held for its hold), 'stopped' on `stop()`
|
|
109
|
+
* (which also calls `stage.release()`, letting a held look go), or 'failed' when the stage threw on a timer */
|
|
110
|
+
export function playCues(stage: CueStage & { release?(): void }, cues: LineCue[], clock?: (() => number) | null): {
|
|
111
|
+
tick(): boolean; stop(): void; readonly done: Promise<'ended' | 'stopped' | 'failed'>; readonly playing: boolean;
|
|
112
|
+
};
|
|
113
|
+
|
|
114
|
+
export function buildLine(input: {
|
|
115
|
+
pcm: Float32Array;
|
|
116
|
+
sampleRate?: number;
|
|
117
|
+
words?: { text: string; start: number; end: number }[] | null;
|
|
118
|
+
/** timed letters (`lettersFromAlignment(tts.alignment)`, or `lettersFromWords(words)` for word timings only): the mouth
|
|
119
|
+
* shaped from the words, lips meeting on m/b/p. buildLine never derives them itself (a clip does) */
|
|
120
|
+
letters?: LineLetter[] | null;
|
|
121
|
+
transcript?: string | null;
|
|
122
|
+
mood?: string | null;
|
|
123
|
+
/** used instead of the derived captions; one that ends past the audio ends with it, one that starts past it is dropped */
|
|
124
|
+
captions?: LineCaption[] | ((duration: number) => LineCaption[]) | null;
|
|
125
|
+
/** an array: the app's direction, merged with the derived cues (`at` on seconds or a word); a function: the whole track */
|
|
126
|
+
cues?: AuthoredCue[] | ((duration: number, captions: LineCaption[]) => LineCue[]) | null;
|
|
127
|
+
turns?: { id: string; start: number; end: number }[] | null;
|
|
128
|
+
}): SpeechLine;
|
|
129
|
+
export function frameCount(duration: number): number;
|
|
130
|
+
export function validateLine(data: unknown): SpeechLine;
|
|
131
|
+
export function sampleLine(line: SpeechLine, time: number): LineSample;
|
|
132
|
+
export function sentenceCaptions(words: LineWord[]): LineCaption[];
|
|
133
|
+
export function lineWords(line: SpeechLine): (LineWord & { estimated?: true })[];
|
|
134
|
+
export function captionWindow(line: SpeechLine, time: number, opts?: { style?: CaptionStyle; reduceMotion?: boolean; maxWords?: number; maxChars?: number; words?: LineWord[] }): CaptionWindow;
|
|
135
|
+
export function applySpeechCue(stage: CueStage, cue: LineCue, id?: unknown, quiet?: boolean): void;
|
|
136
|
+
|
|
137
|
+
/** something with an audio clock: an HTMLAudioElement fits */
|
|
138
|
+
export interface LineAudio { currentTime: number; ended?: boolean; play(): Promise<void> | void; pause(): void }
|
|
139
|
+
export interface LineTick { generation: number; time: number; sample: LineSample; silent: boolean }
|
|
140
|
+
export interface LinePlayer {
|
|
141
|
+
say(line: SpeechLine, opts?: { audio?: LineAudio | null }): number;
|
|
142
|
+
stop(): void;
|
|
143
|
+
readonly generation: number | null;
|
|
144
|
+
readonly silent: boolean;
|
|
145
|
+
tick(): LineTick | null;
|
|
146
|
+
}
|
|
147
|
+
export function createLinePlayer(opts?: {
|
|
148
|
+
onCue?(cue: LineCue, quiet: boolean): void;
|
|
149
|
+
onEnd?(generation: number, reason: 'ended' | 'stopped'): void;
|
|
150
|
+
now?(): number;
|
|
151
|
+
}): LinePlayer;
|
|
152
|
+
|
|
153
|
+
export function cueKind(cue: unknown): 'mood' | 'gesture' | 'brow' | 'look' | null;
|
|
154
|
+
export function validateCues(cues: unknown, duration: number): void;
|
|
155
|
+
export function moodAt(cues: LineCue[], time: number): string;
|
|
156
|
+
export function buildCues(moods: LineCue[], words: { word: string; start: number; end: number }[], extra?: LineCue[], duration?: number): LineCue[];
|
|
157
|
+
export function wordsFromAlignment(alignment: ProviderAlignment | ProviderAlignment[] | null | undefined): { word: string; text: string; start: number; end: number }[];
|
|
158
|
+
/** words in milliseconds (`{ text, startMs, durationMs }`) → timed words, seconds */
|
|
159
|
+
export function wordsFromMs(words: { text: string; startMs: number; durationMs: number }[]): LineWord[];
|
|
160
|
+
export const AZURE_TICKS: number;
|
|
161
|
+
/** Azure Speech WordBoundary events (offset and duration in 100 ns ticks) → timed words; punctuation and sentence boundaries skipped */
|
|
162
|
+
export function wordsFromAzure(events: { text: string; audioOffset: number; duration: number; boundaryType?: string | number }[]): LineWord[];
|
|
163
|
+
/** Amazon Polly speech marks (JSON lines, or their objects) → timed words; each ends where the next starts, the last at `duration` (seconds) */
|
|
164
|
+
export function wordsFromPolly(marks: string | { time: number; type: string; value?: string }[], duration: number): LineWord[];
|
|
165
|
+
/** Cartesia `word_timestamps` (`{ words, start, end }`, seconds; one or a stream's pieces) → timed words */
|
|
166
|
+
export function wordsFromCartesia(timestamps: { words: string[]; start: number[]; end: number[] } | { words: string[]; start: number[]; end: number[] }[]): LineWord[];
|
|
167
|
+
export function analyzePCM(pcm: Float32Array, sampleRate?: number, turns?: { id: string; start: number; end: number }[] | null): { fps: 60; duration: number; frames: LineFrame[] };
|
|
168
|
+
export function analyzePitch(pcm: Float32Array, sampleRate: number, turns?: { id: string; start: number; end: number }[] | null): (number | null)[];
|
|
169
|
+
export function round(n: number): number;
|
|
170
|
+
/** a line's captions sized for a speech bubble (a presenter deck): sentence-sized, beside captionWindow's 4-word phrases */
|
|
171
|
+
export { capsFromLine, captionAt, wordTime, BUBBLE_CAPTION_CHARS, BUBBLE_LIT_HOLD } from './bubble-captions';
|
|
172
|
+
export type { BubbleCaption } from './bubble-captions';
|