@laplace.live/persona-sdk 1.0.0 → 1.2.0

package/dist/values/hands.js

@@ -0,0 +1,89 @@
+/** VTube Studio hand inputs, extended with nizima LIVE's wrist Y rotation. */
+export const HAND_INPUT_NAMES = [
+    'HandLeftFound',
+    'HandLeftPositionX',
+    'HandLeftPositionY',
+    'HandLeftPositionZ',
+    'HandLeftAngleX',
+    'HandLeftAngleY',
+    'HandLeftAngleZ',
+    'HandLeftOpen',
+    'HandLeftFinger_1_Thumb',
+    'HandLeftFinger_2_Index',
+    'HandLeftFinger_3_Middle',
+    'HandLeftFinger_4_Ring',
+    'HandLeftFinger_5_Pinky',
+    'HandRightFound',
+    'HandRightPositionX',
+    'HandRightPositionY',
+    'HandRightPositionZ',
+    'HandRightAngleX',
+    'HandRightAngleY',
+    'HandRightAngleZ',
+    'HandRightOpen',
+    'HandRightFinger_1_Thumb',
+    'HandRightFinger_2_Index',
+    'HandRightFinger_3_Middle',
+    'HandRightFinger_4_Ring',
+    'HandRightFinger_5_Pinky',
+    'BothHandsFound',
+    'HandDistance',
+];
+const HAND_INPUT_SET = new Set(HAND_INPUT_NAMES);
+/** Whether an input uses the canonical hand vocabulary. */
+export function isHandInputName(input) {
+    return HAND_INPUT_SET.has(input);
+}
+/** Names nizima uses for VTS's numbered fingers and both-hands detection. */
+export const NIZIMA_HAND_ALIASES = {
+    HandLeftFingerThumb: 'HandLeftFinger_1_Thumb',
+    HandLeftFingerIndex: 'HandLeftFinger_2_Index',
+    HandLeftFingerMiddle: 'HandLeftFinger_3_Middle',
+    HandLeftFingerRing: 'HandLeftFinger_4_Ring',
+    HandLeftFingerPinky: 'HandLeftFinger_5_Pinky',
+    HandRightFingerThumb: 'HandRightFinger_1_Thumb',
+    HandRightFingerIndex: 'HandRightFinger_2_Index',
+    HandRightFingerMiddle: 'HandRightFinger_3_Middle',
+    HandRightFingerRing: 'HandRightFinger_4_Ring',
+    HandRightFingerPinky: 'HandRightFinger_5_Pinky',
+    HandBothFound: 'BothHandsFound',
+};
+/** Canonicalize a hand name; aliases share canonical units on the plugin API. */
+export function canonicalHandInputName(input) {
+    if (isHandInputName(input))
+        return input;
+    return Object.hasOwn(NIZIMA_HAND_ALIASES, input)
+        ? NIZIMA_HAND_ALIASES[input]
+        : null;
+}
+/** Angles are degrees; positions and distance use VTS's ten-unit scale. */
+export const HAND_INPUT_RANGES = {
+    HandLeftFound: [0, 1],
+    HandLeftPositionX: [-10, 10],
+    HandLeftPositionY: [-10, 10],
+    HandLeftPositionZ: [-10, 10],
+    HandLeftAngleX: [-180, 180],
+    HandLeftAngleY: [-90, 90],
+    HandLeftAngleZ: [-180, 180],
+    HandLeftOpen: [0, 1],
+    HandLeftFinger_1_Thumb: [0, 1],
+    HandLeftFinger_2_Index: [0, 1],
+    HandLeftFinger_3_Middle: [0, 1],
+    HandLeftFinger_4_Ring: [0, 1],
+    HandLeftFinger_5_Pinky: [0, 1],
+    HandRightFound: [0, 1],
+    HandRightPositionX: [-10, 10],
+    HandRightPositionY: [-10, 10],
+    HandRightPositionZ: [-10, 10],
+    HandRightAngleX: [-180, 180],
+    HandRightAngleY: [-90, 90],
+    HandRightAngleZ: [-180, 180],
+    HandRightOpen: [0, 1],
+    HandRightFinger_1_Thumb: [0, 1],
+    HandRightFinger_2_Index: [0, 1],
+    HandRightFinger_3_Middle: [0, 1],
+    HandRightFinger_4_Ring: [0, 1],
+    HandRightFinger_5_Pinky: [0, 1],
+    BothHandsFound: [0, 1],
+    HandDistance: [0, 10],
+};

package/dist/values/lipsync.d.ts

@@ -0,0 +1,106 @@
+export declare const LIP_SYNC_VOWELS: readonly ["A", "I", "U", "E", "O"];
+export type LipSyncVowel = (typeof LIP_SYNC_VOWELS)[number];
+export declare const LIP_SYNC_PHONEMES: readonly ["A", "I", "U", "E", "O", "S"];
+export type LipSyncPhoneme = (typeof LIP_SYNC_PHONEMES)[number];
+export declare const LIP_SYNC_MODES: readonly ["off", "always", "when-untracked"];
+export type LipSyncMode = (typeof LIP_SYNC_MODES)[number];
+/** Public limits, shared by the settings healer, the wire schema, and the panel sliders. */
+export declare const LIP_SYNC_GAIN_MAX = 30;
+export declare const LIP_SYNC_NOISE_GATE_MIN = -60;
+export declare const LIP_SYNC_SMOOTHING_MAX = 0.3;
+/** Local microphone calibration; each sample is a 12-coefficient MFCC vector. */
+export interface LipSyncProfile {
+    deviceId: string;
+    deviceLabel: string;
+    updatedAt: number;
+    samples: Record<LipSyncPhoneme, number[][]>;
+}
+/** Calibration steps that carry no phoneme; `record` is the sixth action and takes one. */
+export declare const LIP_SYNC_CALIBRATION_ACTIONS: readonly ["start", "cancel", "preview", "save", "reset"];
+export type LipSyncCalibrationCommand = {
+    action: (typeof LIP_SYNC_CALIBRATION_ACTIONS)[number];
+} | {
+    action: 'record';
+    phoneme: LipSyncPhoneme;
+};
+export interface LipSyncCalibrationState {
+    phase: 'ready' | 'recording' | 'verifying';
+    phoneme: LipSyncPhoneme | null;
+    progress: number;
+    completed: LipSyncPhoneme[];
+    error: 'too-quiet' | 'insufficient-data' | null;
+}
+/** One desktop microphone, shared by the scene's participating models. */
+export interface LipSyncConfig {
+    enabled: boolean;
+    /** Empty selects the system's default input. */
+    deviceId: string;
+    /** Input gain in dB, 0..30. */
+    gain: number;
+    /** Silence threshold in dBFS, -60..0. */
+    noiseGate: number;
+    /** Volume and vowel smoothing in seconds, 0..0.3. */
+    smoothing: number;
+}
+export declare const DEFAULT_LIP_SYNC_CONFIG: Readonly<LipSyncConfig>;
+/** Analyzer output before model-specific mapping. */
+export interface LipSyncSample {
+    /** Mouth-open envelope, 0..1. */
+    volume: number;
+    /** Smoothed vowel weights sum to at most one; the remainder represents non-vowel audio. */
+    vowels: Record<LipSyncVowel, number>;
+}
+export interface LipSyncState {
+    config: LipSyncConfig;
+    status: 'off' | 'starting' | 'listening' | 'error';
+    error: 'permission' | 'device' | 'analysis' | null;
+    devices: {
+        deviceId: string;
+        label: string;
+    }[];
+    sample: LipSyncSample | null;
+    microphone: {
+        deviceId: string;
+        label: string;
+    } | null;
+    profile: Omit<LipSyncProfile, 'samples'> | null;
+    calibration: LipSyncCalibrationState | null;
+}
+/** Drop incomplete or malformed calibrations rather than feeding them to the analyzer. */
+export declare function healLipSyncProfile(raw: unknown): LipSyncProfile | null;
+/** Heal persisted configuration, retaining supported values within their public limits. */
+export declare function healLipSyncConfig(raw: unknown): LipSyncConfig;
+/** Whether a persisted per-model mode is supported. */
+export declare function isLipSyncMode(value: unknown): value is LipSyncMode;
+export declare const VOICE_INPUT_NAMES: readonly ["VoiceVolume", "VoiceFrequency", "VoiceVolumePlusMouthOpen", "VoiceFrequencyPlusMouthSmile", "VoiceA", "VoiceI", "VoiceU", "VoiceE", "VoiceO", "VoiceSilence", "VoiceMouthOpen", "VoiceMouthSpread"];
+export type VoiceInputName = (typeof VOICE_INPUT_NAMES)[number];
+export declare const VOICE_INPUT_RANGES: {
+    readonly VoiceVolume: readonly [0, 1];
+    readonly VoiceFrequency: readonly [0, 1];
+    readonly VoiceVolumePlusMouthOpen: readonly [0, 2];
+    readonly VoiceFrequencyPlusMouthSmile: readonly [0, 1];
+    readonly VoiceA: readonly [0, 1];
+    readonly VoiceI: readonly [0, 1];
+    readonly VoiceU: readonly [0, 1];
+    readonly VoiceE: readonly [0, 1];
+    readonly VoiceO: readonly [0, 1];
+    readonly VoiceSilence: readonly [0, 1];
+    readonly VoiceMouthOpen: readonly [0, 1];
+    readonly VoiceMouthSpread: readonly [-1, 1];
+};
+/** Semantic import aliases; source analyzer amplitudes need not match Persona's. */
+export declare const NIZIMA_VOICE_ALIASES: {
+    readonly LipSyncVolume: "VoiceVolume";
+    readonly LipSyncMouthOpen: "VoiceMouthOpen";
+    readonly LipSyncMouthSpread: "VoiceMouthSpread";
+    readonly LipSyncMouthA: "VoiceA";
+    readonly LipSyncMouthI: "VoiceI";
+    readonly LipSyncMouthU: "VoiceU";
+    readonly LipSyncMouthE: "VoiceE";
+    readonly LipSyncMouthO: "VoiceO";
+};
+/**
+ * VTS-compatible binding inputs. Frequency projects vowel weights onto its conventional
+ * mouth-form axis; composite sums retain over-range values for the authored binding to clamp.
+ */
+export declare function voiceInputs(sample: LipSyncSample, faceInputs?: Readonly<Partial<Record<'MouthOpen' | 'MouthSmile', number>>>): Record<VoiceInputName, number>;

package/dist/values/lipsync.js

@@ -0,0 +1,120 @@
+import { finiteOr, isFiniteNumber, isRecord, nonEmptyString } from "./guards.js";
+import { clamp } from "./limits.js";
+export const LIP_SYNC_VOWELS = ['A', 'I', 'U', 'E', 'O'];
+export const LIP_SYNC_PHONEMES = [...LIP_SYNC_VOWELS, 'S'];
+export const LIP_SYNC_MODES = ['off', 'always', 'when-untracked'];
+/** Public limits, shared by the settings healer, the wire schema, and the panel sliders. */
+export const LIP_SYNC_GAIN_MAX = 30;
+export const LIP_SYNC_NOISE_GATE_MIN = -60;
+export const LIP_SYNC_SMOOTHING_MAX = 0.3;
+/** Calibration steps that carry no phoneme; `record` is the sixth action and takes one. */
+export const LIP_SYNC_CALIBRATION_ACTIONS = ['start', 'cancel', 'preview', 'save', 'reset'];
+export const DEFAULT_LIP_SYNC_CONFIG = {
+    enabled: false,
+    deviceId: '',
+    gain: 0,
+    noiseGate: -45,
+    smoothing: 0.08,
+};
+/** Drop incomplete or malformed calibrations rather than feeding them to the analyzer. */
+export function healLipSyncProfile(raw) {
+    if (!isRecord(raw))
+        return null;
+    const deviceId = nonEmptyString(raw.deviceId);
+    if (deviceId === null ||
+        typeof raw.deviceLabel !== 'string' ||
+        !isFiniteNumber(raw.updatedAt) ||
+        raw.updatedAt < 0 ||
+        !isRecord(raw.samples)) {
+        return null;
+    }
+    const samples = { A: [], I: [], U: [], E: [], O: [], S: [] };
+    for (const phoneme of LIP_SYNC_PHONEMES) {
+        const vectors = raw.samples[phoneme];
+        if (!Array.isArray(vectors) || vectors.length < 12 || vectors.length > 40)
+            return null;
+        for (const vector of vectors) {
+            if (!Array.isArray(vector) || vector.length !== 12 || !vector.every(isFiniteNumber))
+                return null;
+            samples[phoneme].push([...vector]);
+        }
+    }
+    return { deviceId, deviceLabel: raw.deviceLabel, updatedAt: raw.updatedAt, samples };
+}
+/** Heal persisted configuration, retaining supported values within their public limits. */
+export function healLipSyncConfig(raw) {
+    const value = isRecord(raw) ? raw : {};
+    return {
+        enabled: typeof value.enabled === 'boolean' ? value.enabled : DEFAULT_LIP_SYNC_CONFIG.enabled,
+        deviceId: typeof value.deviceId === 'string' ? value.deviceId : DEFAULT_LIP_SYNC_CONFIG.deviceId,
+        gain: clamp(finiteOr(value.gain, DEFAULT_LIP_SYNC_CONFIG.gain), 0, LIP_SYNC_GAIN_MAX),
+        noiseGate: clamp(finiteOr(value.noiseGate, DEFAULT_LIP_SYNC_CONFIG.noiseGate), LIP_SYNC_NOISE_GATE_MIN, 0),
+        smoothing: clamp(finiteOr(value.smoothing, DEFAULT_LIP_SYNC_CONFIG.smoothing), 0, LIP_SYNC_SMOOTHING_MAX),
+    };
+}
+/** Whether a persisted per-model mode is supported. */
+export function isLipSyncMode(value) {
+    return LIP_SYNC_MODES.some(mode => mode === value);
+}
+export const VOICE_INPUT_NAMES = [
+    'VoiceVolume',
+    'VoiceFrequency',
+    'VoiceVolumePlusMouthOpen',
+    'VoiceFrequencyPlusMouthSmile',
+    'VoiceA',
+    'VoiceI',
+    'VoiceU',
+    'VoiceE',
+    'VoiceO',
+    'VoiceSilence',
+    'VoiceMouthOpen',
+    'VoiceMouthSpread',
+];
+export const VOICE_INPUT_RANGES = {
+    VoiceVolume: [0, 1],
+    VoiceFrequency: [0, 1],
+    VoiceVolumePlusMouthOpen: [0, 2],
+    VoiceFrequencyPlusMouthSmile: [0, 1],
+    VoiceA: [0, 1],
+    VoiceI: [0, 1],
+    VoiceU: [0, 1],
+    VoiceE: [0, 1],
+    VoiceO: [0, 1],
+    VoiceSilence: [0, 1],
+    VoiceMouthOpen: [0, 1],
+    VoiceMouthSpread: [-1, 1],
+};
+/** Semantic import aliases; source analyzer amplitudes need not match Persona's. */
+export const NIZIMA_VOICE_ALIASES = {
+    LipSyncVolume: 'VoiceVolume',
+    LipSyncMouthOpen: 'VoiceMouthOpen',
+    LipSyncMouthSpread: 'VoiceMouthSpread',
+    LipSyncMouthA: 'VoiceA',
+    LipSyncMouthI: 'VoiceI',
+    LipSyncMouthU: 'VoiceU',
+    LipSyncMouthE: 'VoiceE',
+    LipSyncMouthO: 'VoiceO',
+};
+/**
+ * VTS-compatible binding inputs. Frequency projects vowel weights onto its conventional
+ * mouth-form axis; composite sums retain over-range values for the authored binding to clamp.
+ */
+export function voiceInputs(sample, faceInputs) {
+    const { volume, vowels } = sample;
+    // Vowel projection, not measured pitch: https://github.com/DenchiSoft/VTubeStudio/wiki/Lipsync
+    const frequency = 0.5 + (volume * (-0.9 * vowels.A - 0.7 * vowels.I - 0.6 * vowels.U + vowels.E + 0.9 * vowels.O)) / 2;
+    return {
+        VoiceVolume: volume,
+        VoiceFrequency: frequency,
+        VoiceVolumePlusMouthOpen: volume + (faceInputs?.MouthOpen ?? 0),
+        VoiceFrequencyPlusMouthSmile: faceInputs?.MouthSmile === undefined ? frequency : faceInputs.MouthSmile + (frequency - 0.5) * 2,
+        VoiceA: volume * vowels.A,
+        VoiceI: volume * vowels.I,
+        VoiceU: volume * vowels.U,
+        VoiceE: volume * vowels.E,
+        VoiceO: volume * vowels.O,
+        VoiceSilence: 1 - volume,
+        VoiceMouthOpen: volume * (vowels.A + 0.4 * vowels.I + 0.4 * vowels.U + 0.7 * vowels.E + vowels.O),
+        VoiceMouthSpread: volume * (vowels.A + vowels.I - vowels.U + vowels.E - vowels.O),
+    };
+}

package/dist/wire/events.d.ts

@@ -14,7 +14,7 @@ export interface EventMap {
     'shortcut.state': {
         shortcuts: ShortcutInfo[];
     };
-    /** `sources` maps source id → status; optional so pre-multi-source servers still parse. */
+    /** Network channel aggregates; `sources` includes webcam status and is optional for older servers. */
     'tracking.status': {
         tracking: TrackingStatus;
         pose: PoseStatus;

package/dist/wire/methods.d.ts

@@ -1,8 +1,9 @@
 import type { Binding, BindingEditorData } from '../values/bindings.ts';
 import type { ControllerMovementConfig, ControllerState } from '../values/controller.ts';
+import type { LipSyncCalibrationCommand, LipSyncConfig, LipSyncMode, LipSyncState } from '../values/lipsync.ts';
 import type { SceneInspection } from '../values/stage-info.ts';
 import type { EventName } from './events.ts';
-import type { AnchorOption, AppCapability, AssetKind, AssetRef, Attach, AttachHeadAngle, BindingInput, Expression, ExpressionPersistence, HotkeyConfig, HotkeyState, InjectEntry, InjectTarget, InstanceRuntime, JsonValue, ModelInfo, ModelRef, MotionGroup, MToonTuning, ObjectContent, ObjectLightOverride, ObjectSpace, Place2D, Place3D, PlayingMotion, PoseSourceId, PoseStatus, Scene, SceneItem, ScenePatch, SceneState, ScreenPlacement, Settings, SettingsPatch, ShortcutInfo, TrackingSourceConfig, TrackingSourceId, TrackingSourceKind, TrackingStatus, VrmPlacement } from './types.ts';
+import type { AnchorOption, AppCapability, AssetKind, AssetRef, Attach, AttachHeadAngle, BindingInput, Expression, ExpressionPersistence, HandTrackingMode, HotkeyConfig, HotkeyState, InjectEntry, InjectTarget, InstanceRuntime, JsonValue, LayerEffectKey, LayerEffects, LayerEffectsPatch, MediaPipeConfig, ModelInfo, ModelRef, MotionGroup, MToonTuning, ObjectContent, ObjectLightOverride, ObjectSpace, Place2D, Place3D, PlayingMotion, PoseSourceId, PoseStatus, Scene, SceneItem, ScenePatch, SceneState, ScreenPlacement, Settings, SettingsPatch, ShortcutInfo, TrackingSourceConfig, TrackingSourceId, TrackingSourceKind, TrackingStatus, VrmPlacement } from './types.ts';
 /** Marker for methods that take no parameters; the client lets you omit the argument. */
 export type EmptyRequest = Record<never, never>;
 /** Marker for methods whose success response carries no data. */
@@ -76,11 +77,24 @@ export interface InstanceSetVisibleRequest {
     visible: boolean;
 }
 export type InstanceSetVisibleResponse = EmptyResponse;
+/** Edits one model or object's source effects; gate on the `layer-effects` app capability. */
+export interface InstanceSetEffectsRequest {
+    instanceId: string;
+    effects: LayerEffectsPatch;
+    /** Replaces the added-effect list; omitted preserves it. Enabled effects always appear. */
+    effectLayers?: LayerEffectKey[];
+}
+export interface InstanceSetEffectsResponse {
+    effects: LayerEffects;
+    effectLayers: LayerEffectKey[];
+}
 /** Omitted fields keep their binding; null stops tracking that channel for this instance. */
 export interface InstanceSetTrackingSourcesRequest {
     instanceId: string;
     faceSourceId?: string | null;
     poseSourceId?: string | null;
+    handSourceId?: string | null;
+    handTrackingMode?: HandTrackingMode;
 }
 export type InstanceSetTrackingSourcesResponse = EmptyResponse;
 export interface InstanceSetControllerMovementRequest {
@@ -90,6 +104,21 @@ export interface InstanceSetControllerMovementRequest {
 export interface InstanceSetControllerMovementResponse {
     movement: ControllerMovementConfig;
 }
+export interface InstanceSetLipSyncRequest {
+    instanceId?: string;
+    mode: LipSyncMode;
+}
+export interface InstanceSetLipSyncResponse {
+    mode: LipSyncMode;
+}
+export type LipSyncStateRequest = EmptyRequest;
+export type LipSyncStateResponse = LipSyncState;
+export type LipSyncConfigureRequest = Partial<LipSyncConfig>;
+export type LipSyncConfigureResponse = LipSyncConfig;
+export type LipSyncRestartRequest = EmptyRequest;
+export type LipSyncRestartResponse = EmptyResponse;
+export type LipSyncCalibrateRequest = LipSyncCalibrationCommand;
+export type LipSyncCalibrateResponse = LipSyncState;
 export interface InstanceReorderRequest {
     /** Full z-order within the scene; ids not listed keep their relative order at the end. */
     orderedInstanceIds: string[];
@@ -311,6 +340,7 @@ export interface SettingsPatchResponse {
     settings: Settings;
 }
 export interface TrackingSetEnabledRequest {
+    /** Enables network face sources; webcam tracking is controlled by its source's enabled flag. */
     enabled: boolean;
 }
 export type TrackingSetEnabledResponse = EmptyResponse;
@@ -319,6 +349,7 @@ export interface TrackingSetSourceRequest {
 }
 export type TrackingSetSourceResponse = EmptyResponse;
 export interface PoseSetEnabledRequest {
+    /** Enables network body sources; webcam tracking is controlled by its source's enabled flag. */
     enabled: boolean;
 }
 export type PoseSetEnabledResponse = EmptyResponse;
@@ -343,6 +374,7 @@ export interface TrackingAddSourceRequest {
      * ifacialmocap: the receive port, default 49983 (set the phone's Send Port to match).
      */
     port?: number;
+    mediapipe?: Partial<MediaPipeConfig>;
 }
 export interface TrackingAddSourceResponse {
     source: TrackingSourceConfig;
@@ -353,6 +385,7 @@ export interface TrackingUpdateSourceRequest {
     enabled?: boolean;
     phoneIp?: string | null;
     port?: number;
+    mediapipe?: Partial<MediaPipeConfig>;
 }
 export interface TrackingUpdateSourceResponse {
     source: TrackingSourceConfig;
@@ -363,7 +396,9 @@ export interface TrackingRemoveSourceRequest {
 export type TrackingRemoveSourceResponse = EmptyResponse;
 export type TrackingStatusRequest = EmptyRequest;
 export interface TrackingStatusResponse {
+    /** Aggregate network face status. Webcam activity is reported in sources. */
     tracking: TrackingStatus;
+    /** Aggregate network body status. Webcam activity is reported in sources. */
     pose: PoseStatus;
     /** Per-source status by {@link TrackingSourceConfig.id}. */
     sources: Record<string, TrackingStatus>;
@@ -374,8 +409,6 @@ export interface StageResetTransformRequest {
 export type StageResetTransformResponse = EmptyResponse;
 export type StageResetCameraRequest = EmptyRequest;
 export type StageResetCameraResponse = EmptyResponse;
-export type StageShockwaveRequest = EmptyRequest;
-export type StageShockwaveResponse = EmptyResponse;
 export type AppInfoRequest = EmptyRequest;
 export interface AppInfoResponse {
     name: string;
@@ -567,6 +600,26 @@ export interface MethodMap {
         request: InstanceSetBreathRequest;
         response: InstanceSetBreathResponse;
     };
+    'instance.setLipSync': {
+        request: InstanceSetLipSyncRequest;
+        response: InstanceSetLipSyncResponse;
+    };
+    'lipSync.state': {
+        request: LipSyncStateRequest;
+        response: LipSyncStateResponse;
+    };
+    'lipSync.configure': {
+        request: LipSyncConfigureRequest;
+        response: LipSyncConfigureResponse;
+    };
+    'lipSync.restart': {
+        request: LipSyncRestartRequest;
+        response: LipSyncRestartResponse;
+    };
+    'lipSync.calibrate': {
+        request: LipSyncCalibrateRequest;
+        response: LipSyncCalibrateResponse;
+    };
     'controller.state': {
         request: ControllerStateRequest;
         response: ControllerStateResponse;
@@ -650,6 +703,10 @@ export interface MethodMap {
         request: InstanceSetVisibleRequest;
         response: InstanceSetVisibleResponse;
     };
+    'instance.setEffects': {
+        request: InstanceSetEffectsRequest;
+        response: InstanceSetEffectsResponse;
+    };
     /** Bind this instance's face/pose channels to tracking sources (null stops tracking that channel). */
     'instance.setTrackingSources': {
         request: InstanceSetTrackingSourcesRequest;
@@ -864,10 +921,6 @@ export interface MethodMap {
         request: StageResetCameraRequest;
         response: StageResetCameraResponse;
     };
-    'stage.shockwave': {
-        request: StageShockwaveRequest;
-        response: StageShockwaveResponse;
-    };
     'app.info': {
         request: AppInfoRequest;
         response: AppInfoResponse;

package/dist/wire/protocol.d.ts

@@ -1,6 +1,6 @@
 import type { InjectTarget } from './types.ts';
 /** Bumped on breaking wire changes; the server reports it in `hello` and clients warn on mismatch. */
-export declare const PROTOCOL_VERSION = 1;
+export declare const PROTOCOL_VERSION = 3;
 /** Default port the Persona API server listens on (user-configurable in the app). */
 export declare const DEFAULT_API_PORT = 25034;
 /** Default host clients dial: the server binds loopback unless LAN is opted in. */

package/dist/wire/protocol.js

@@ -1,5 +1,5 @@
 /** Bumped on breaking wire changes; the server reports it in `hello` and clients warn on mismatch. */
-export const PROTOCOL_VERSION = 1;
+export const PROTOCOL_VERSION = 3;
 /** Default port the Persona API server listens on (user-configurable in the app). */
 export const DEFAULT_API_PORT = 25034;
 /** Default host clients dial: the server binds loopback unless LAN is opted in. */

package/dist/wire/schemas.d.ts

@@ -22,6 +22,13 @@ export declare const InjectEntrySchema: z.ZodObject<{
     weight: z.ZodOptional<z.ZodNumber>;
 }, z.core.$strip>;
 export declare const SettingsPatchSchema: z.ZodObject<{
+    lipSync: z.ZodOptional<z.ZodObject<{
+        enabled: z.ZodOptional<z.ZodBoolean>;
+        deviceId: z.ZodOptional<z.ZodString>;
+        gain: z.ZodOptional<z.ZodNumber>;
+        noiseGate: z.ZodOptional<z.ZodNumber>;
+        smoothing: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strip>>;
     controller: z.ZodOptional<z.ZodObject<{
         enabled: z.ZodOptional<z.ZodBoolean>;
     }, z.core.$strip>>;
@@ -36,9 +43,9 @@ export declare const SettingsPatchSchema: z.ZodObject<{
         fpsLimit: z.ZodOptional<z.ZodNumber>;
         selectionOutline: z.ZodOptional<z.ZodBoolean>;
         effectsQuality: z.ZodOptional<z.ZodEnum<{
-            high: "high";
             low: "low";
             medium: "medium";
+            high: "high";
         }>>;
         renderScale: z.ZodOptional<z.ZodNumber>;
         live2dEngine: z.ZodOptional<z.ZodEnum<{
@@ -48,6 +55,42 @@ export declare const SettingsPatchSchema: z.ZodObject<{
     }, z.core.$strip>>;
 }, z.core.$strip>;
 export declare const requestSchemas: {
+    'lipSync.state': z.ZodObject<{}, z.core.$strip>;
+    'lipSync.configure': z.ZodObject<{
+        enabled: z.ZodOptional<z.ZodBoolean>;
+        deviceId: z.ZodOptional<z.ZodString>;
+        gain: z.ZodOptional<z.ZodNumber>;
+        noiseGate: z.ZodOptional<z.ZodNumber>;
+        smoothing: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strip>;
+    'lipSync.restart': z.ZodObject<{}, z.core.$strip>;
+    'lipSync.calibrate': z.ZodDiscriminatedUnion<[z.ZodObject<{
+        action: z.ZodEnum<{
+            start: "start";
+            cancel: "cancel";
+            preview: "preview";
+            save: "save";
+            reset: "reset";
+        }>;
+    }, z.core.$strip>, z.ZodObject<{
+        action: z.ZodLiteral<"record">;
+        phoneme: z.ZodEnum<{
+            A: "A";
+            I: "I";
+            U: "U";
+            E: "E";
+            O: "O";
+            S: "S";
+        }>;
+    }, z.core.$strip>], "action">;
+    'instance.setLipSync': z.ZodObject<{
+        instanceId: z.ZodOptional<z.ZodString>;
+        mode: z.ZodEnum<{
+            off: "off";
+            always: "always";
+            "when-untracked": "when-untracked";
+        }>;
+    }, z.core.$strip>;
     'registry.thumbnail': z.ZodObject<{
         kind: z.ZodEnum<{
             model: "model";
@@ -131,10 +174,22 @@ export declare const requestSchemas: {
             "vts-ios": "vts-ios";
             vmc: "vmc";
             mocopi: "mocopi";
+            mediapipe: "mediapipe";
         }>;
         name: z.ZodOptional<z.ZodString>;
         phoneIp: z.ZodOptional<z.ZodNullable<z.ZodString>>;
         port: z.ZodOptional<z.ZodNumber>;
+        mediapipe: z.ZodOptional<z.ZodObject<{
+            deviceId: z.ZodOptional<z.ZodString>;
+            mirror: z.ZodOptional<z.ZodBoolean>;
+            face: z.ZodOptional<z.ZodBoolean>;
+            hands: z.ZodOptional<z.ZodBoolean>;
+            body: z.ZodOptional<z.ZodBoolean>;
+            delegate: z.ZodOptional<z.ZodEnum<{
+                CPU: "CPU";
+                GPU: "GPU";
+            }>>;
+        }, z.core.$strip>>;
     }, z.core.$strip>;
     'tracking.updateSource': z.ZodObject<{
         id: z.ZodString;
@@ -142,6 +197,17 @@ export declare const requestSchemas: {
         enabled: z.ZodOptional<z.ZodBoolean>;
         phoneIp: z.ZodOptional<z.ZodNullable<z.ZodString>>;
         port: z.ZodOptional<z.ZodNumber>;
+        mediapipe: z.ZodOptional<z.ZodObject<{
+            deviceId: z.ZodOptional<z.ZodString>;
+            mirror: z.ZodOptional<z.ZodBoolean>;
+            face: z.ZodOptional<z.ZodBoolean>;
+            hands: z.ZodOptional<z.ZodBoolean>;
+            body: z.ZodOptional<z.ZodBoolean>;
+            delegate: z.ZodOptional<z.ZodEnum<{
+                CPU: "CPU";
+                GPU: "GPU";
+            }>>;
+        }, z.core.$strip>>;
     }, z.core.$strip>;
     'tracking.removeSource': z.ZodObject<{
         id: z.ZodString;
@@ -152,17 +218,24 @@ export declare const requestSchemas: {
     'asset.register': z.ZodObject<{
         path: z.ZodString;
         want: z.ZodOptional<z.ZodEnum<{
-            lut: "lut";
             image: "image";
             video: "video";
             prop: "prop";
             ibl: "ibl";
+            lut: "lut";
             animation: "animation";
             cameraMotion: "cameraMotion";
         }>>;
     }, z.core.$strip>;
     'settings.patch': z.ZodObject<{
         settings: z.ZodObject<{
+            lipSync: z.ZodOptional<z.ZodObject<{
+                enabled: z.ZodOptional<z.ZodBoolean>;
+                deviceId: z.ZodOptional<z.ZodString>;
+                gain: z.ZodOptional<z.ZodNumber>;
+                noiseGate: z.ZodOptional<z.ZodNumber>;
+                smoothing: z.ZodOptional<z.ZodNumber>;
+            }, z.core.$strip>>;
             controller: z.ZodOptional<z.ZodObject<{
                 enabled: z.ZodOptional<z.ZodBoolean>;
             }, z.core.$strip>>;
@@ -177,9 +250,9 @@ export declare const requestSchemas: {
                 fpsLimit: z.ZodOptional<z.ZodNumber>;
                 selectionOutline: z.ZodOptional<z.ZodBoolean>;
                 effectsQuality: z.ZodOptional<z.ZodEnum<{
-                    high: "high";
                     low: "low";
                     medium: "medium";
+                    high: "high";
                 }>>;
                 renderScale: z.ZodOptional<z.ZodNumber>;
                 live2dEngine: z.ZodOptional<z.ZodEnum<{