mesh-avatar 0.2.0 → 0.3.0

package/README.md

@@ -2,6 +2,17 @@
 
 A WebGL runtime for 2D mesh avatars created with [Mesh Avatar Studio](https://github.com/shinshin86/mesh-avatar-studio). Load a `.mavatar` archive or an avatar folder into a canvas, then control its motion, voice level and expressions.
 
+For React and Next.js, use [mesh-avatar-react](https://github.com/shinshin86/mesh-avatar-studio/tree/main/packages/react).
+
+<p align="center">
+  <img src="https://raw.githubusercontent.com/shinshin86/mesh-avatar-studio/main/docs/images/face-tracking.gif" width="400" alt="A Mesh Avatar Studio avatar following a face through a webcam">
+</p>
+
+To move an avatar with your own face and voice without installing anything, open the
+[Player](https://shinshin86.github.io/mesh-avatar-studio/play/) in Chrome or Edge.
+
+To embed an avatar with HTML, use [mesh-avatar-element](https://github.com/shinshin86/mesh-avatar-studio/tree/main/packages/element). It provides a `<mesh-avatar>` custom element and a standalone CDN script.
+
 ## Install
 
 ```sh
@@ -77,6 +88,38 @@ Emotion tags are `happy`, `sad`, `angry`, `surprised`, `relaxed` and `neutral` (
 
 For custom loading or tooling, the package also exports `createMeshAvatar`, `openAvatar`, `packAvatar`, `unpackAvatar`, `validateAvatarFiles`, `parseManifest`, `parseRig`, `validateRig` and lighting settings. See the [avatar format reference](https://github.com/shinshin86/mesh-avatar-studio/blob/main/docs/format.md) for archive contents, validation and compatibility rules.
 
+## Mouth movement
+
+Three optional settings control audio loudness lip sync and kana previews. `granularity`
+changes how readily the mouth separates syllables, `openness` changes the opening amount,
+and `closeSpeed` changes how quickly it closes. Each accepts a finite value from 0 to 1;
+0.5 preserves the existing behavior.
+
+The Studio's five styles use these values:
+
+| Style | `granularity` | `openness` | `closeSpeed` |
+| --- | --- | --- | --- |
+| Relaxed | 0 | 0.5 | 0 |
+| Standard | 0.5 | 0.5 | 0.5 |
+| Snappy | 1 | 0.5 | 1 |
+| Subtle | 0.5 | 0 | 0.5 |
+| Expressive | 0.5 | 1 | 0.5 |
+
+If your app already supplies a rig and built assets, pass `lipSync` when creating the avatar:
+
+```ts
+import { createMeshAvatar, type LipSyncSettings } from 'mesh-avatar';
+
+const lipSync: LipSyncSettings = { granularity: 0, openness: 0.5, closeSpeed: 0 };
+const avatar = await createMeshAvatar(canvas, { rig, assets, lipSync });
+avatar.setLipSync({ openness: 0.75 }); // Other settings keep their current values.
+```
+
+Avatars saved or exported in Studio carry these settings in `avatar.json.lipSync`.
+`loadMeshAvatar` applies them automatically; explicit `options.lipSync` fields override the
+saved values. `setLipSync` is a partial update and works during playback. The optional
+`preset` field records the editor selection only; playback uses the three numbers.
+
 ## License
 
 The runtime is MIT licensed; see [LICENSE](./LICENSE). The browser bundle includes [fflate](https://github.com/101arrowz/fflate), with its MIT notice retained in the bundle.

package/dist/engine/createMeshAvatar.d.ts

@@ -10,6 +10,8 @@ export function createMeshAvatarImpl(canvas: HTMLCanvasElement, options: import(
     getParameters(): {};
     /** 0..1 loudness of the voice being played (e.g. normalised RMS). */
     setVoiceLevel(v: any): void;
+    /** Partial mouth-movement settings; omitted fields retain their current values. */
+    setLipSync(settings: any): void;
     /** true while TTS audio is playing: idle motions pause and the head nods along. */
     setSpeaking(on: any): void;
     /** AITuber OnAir emotion tag: happy / sad / angry / surprised / relaxed / neutral (or null). */

package/dist/engine/index.d.ts

@@ -1,6 +1,7 @@
 import type { LightingSettings } from '../lighting/settings.js';
 import type { Rig } from '../rig/types.js';
 import { OVERLAY_EXPRESSIONS } from './expression-overlay.js';
+import type { LipSyncSettings } from './lip-sync.js';
 export type LiveExpression = keyof typeof OVERLAY_EXPRESSIONS;
 export declare const LIVE_EXPRESSIONS: LiveExpression[];
 export interface MeshAvatarOptions {
@@ -12,6 +13,7 @@ export interface MeshAvatarOptions {
     padSide?: number;
     fit?: 'contain' | 'cover';
     preserveMouthForm?: boolean;
+    lipSync?: LipSyncSettings;
 }
 export interface MeshAvatar {
     readonly motions: {
@@ -28,6 +30,8 @@ export interface MeshAvatar {
     setParameters(parameters: Record<string, number>, weight?: number): void;
     getParameters(): Record<string, number>;
     setVoiceLevel(value: number): void;
+    /** Partial mouth-movement settings; omitted fields retain their current values. */
+    setLipSync(settings: LipSyncSettings): void;
     setSpeaking(on: boolean): void;
     setEmotion(tag: string | null, options?: {
         playMotion?: boolean;

package/dist/engine/kana.d.ts

@@ -7,7 +7,7 @@ export function kanaToMoras(text: any, mora?: number): {
     onset: string | null;
 }[];
 /** Mouth target at time t (s) of a mora timeline, or null when it is over. */
-export function sampleMoras(moras: any, t: any): {
+export function sampleMoras(moras: any, t: any, granularity?: number): {
     open: number;
     form: any;
     vowel: any;

package/dist/engine/lip-sync.d.ts

@@ -0,0 +1,21 @@
+export interface LipSyncSettings {
+    granularity?: number;
+    openness?: number;
+    closeSpeed?: number;
+    /** Editor selection only; playback uses the numeric settings. */
+    preset?: 'calm' | 'default' | 'lively' | 'subtle' | 'big' | 'custom';
+}
+type ResolvedLipSync = Required<Pick<LipSyncSettings, 'granularity' | 'openness' | 'closeSpeed'>>;
+export declare function parseLipSyncSettings(input: unknown): LipSyncSettings;
+export declare function resolveLipSync(input?: LipSyncSettings, base?: ResolvedLipSync): ResolvedLipSync;
+export declare function lipMotionOptions(settings: ResolvedLipSync): {
+    arm: number;
+    trigger: number;
+    relax: number;
+    range: number;
+    risePower: number;
+    openness: number;
+    audioRelease: number;
+    kanaRelease: number;
+};
+export {};

package/dist/engine/motion.d.ts

@@ -192,6 +192,17 @@ export class Motion {
     mouth: number;
     vowel: string;
     vowelForm: number;
+    lipSync: Required<Pick<import("./lip-sync.js").LipSyncSettings, "granularity" | "openness" | "closeSpeed">>;
+    lipMotion: {
+        arm: number;
+        trigger: number;
+        relax: number;
+        range: number;
+        risePower: number;
+        openness: number;
+        audioRelease: number;
+        kanaRelease: number;
+    };
     syllableArmed: boolean;
     voicePeak: number;
     voiceTrough: number;
@@ -221,6 +232,7 @@ export class Motion {
     setExpression(k: any): void;
     hasMotion(id: any): boolean;
     setVoiceLevel(v: any): void;
+    setLipSync(settings: any): void;
     setSpeaking(on: any): void;
     /** `playMotion: false` changes only the face (a caller that picks motions itself). */
     setEmotion(tag: any, { playMotion }?: {

package/dist/format/manifest.d.ts

@@ -1,3 +1,4 @@
+import { type LipSyncSettings } from '../engine/lip-sync.js';
 export interface AvatarManifest {
     format: 'mesh-avatar';
     version: 1;
@@ -9,6 +10,7 @@ export interface AvatarManifest {
     studio?: {
         version: string;
     };
+    lipSync?: LipSyncSettings;
 }
 export declare function parseManifest(input: unknown): AvatarManifest;
 export declare function defaultManifest(name: string): AvatarManifest;

package/dist/format/project-archive.d.ts

@@ -1,6 +1,8 @@
+import type { LipSyncSettings } from '../engine/lip-sync.js';
 type Files = Record<string, Uint8Array>;
-export declare function packProjectAvatar(read: (paths: Set<string>) => Promise<Files>, { name, includeSource }: {
+export declare function packProjectAvatar(read: (paths: Set<string>) => Promise<Files>, { name, includeSource, lipSync }: {
     name: string;
     includeSource?: boolean;
+    lipSync?: LipSyncSettings;
 }): Promise<Uint8Array>;
 export {};

package/dist/index.d.ts

@@ -1,5 +1,6 @@
 export { createMeshAvatar, LIVE_EXPRESSIONS } from './engine/index.js';
 export type { MeshAvatar, MeshAvatarOptions, LiveExpression } from './engine/index.js';
+export type { LipSyncSettings } from './engine/lip-sync.js';
 export type { Rig, Eye, Strand, Accessory, Hand, Ellipse, Point, Band } from './rig/types.js';
 export { validateRig, parseRig } from './rig/validate.js';
 export { PARAMS } from './engine/rig.js';