@laplace.live/persona-sdk 0.18.1 → 1.1.0

package/dist/wire/types.js

@@ -2,6 +2,9 @@
 // carry. Structural mirrors of the app's scene/settings models, minus anything
 // filesystem-shaped — model refs are sanitized to ids, never directories.
 import { ARKIT_INPUT_NAMES } from "../values/arkit.js";
+import { BASE_CONTROLLER_INPUT_NAMES, BASE_CONTROLLER_INPUT_RANGES, controllerInputRange, isControllerInputName, } from "../values/controller.js";
+import { HAND_INPUT_NAMES, HAND_INPUT_RANGES } from "../values/hands.js";
+import { VOICE_INPUT_NAMES, VOICE_INPUT_RANGES, } from "../values/lipsync.js";
 /**
  * How a registered file is labelled. Wider than an object's content kinds: `.hdr` is only
  * ever an environment map, and a `.vmd` splits by content — `cameraMotion` for one that
@@ -23,7 +26,15 @@ export const SHADOW_QUALITY_LEVELS = ['off', 'low', 'medium', 'high', 'ultra'];
  * App-level features a client gates on (never version-sniff): `hello` and
  * `app.info` report them — the per-app mirror of {@link InstanceRuntime.capabilities}.
  */
-export const APP_CAPABILITIES = ['storage', 'speech', 'shortcuts'];
+export const APP_CAPABILITIES = [
+    'storage',
+    'speech',
+    'shortcuts',
+    'controllers',
+    'model-editing',
+    'asset-inspection',
+    'layer-effects',
+];
 export function isAppCapability(v) {
     return typeof v === 'string' && APP_CAPABILITIES.includes(v);
 }
@@ -38,15 +49,69 @@ export const SHORTCUT_ACTION_KINDS = [
     'stream-mode',
 ];
 // ---- Settings ------------------------------------------------------------------
-export const TRACKING_SOURCE_IDS = ['vts-ios', 'ifacialmocap'];
+export const TRACKING_SOURCE_IDS = ['persona-ios', 'ifacialmocap', 'vts-ios'];
 /** Body-pose protocols. Every one so far is UDP with a configurable port. */
-export const POSE_SOURCE_IDS = ['vmc'];
-/** Every protocol a tracking source instance can speak; face kinds plus the body (vmc) kind. */
-export const TRACKING_SOURCE_KINDS = [...TRACKING_SOURCE_IDS, ...POSE_SOURCE_IDS];
-/** Whether a source kind feeds the face channel (vmc is the body channel). */
+export const POSE_SOURCE_IDS = ['vmc', 'mocopi'];
+/** Every protocol a tracking source instance can speak; face and body kinds. */
+export const TRACKING_SOURCE_KINDS = [...TRACKING_SOURCE_IDS, ...POSE_SOURCE_IDS, 'mediapipe'];
+export const TRACKING_CHANNELS = ['face', 'pose', 'hands'];
+/** The scene-item binding each channel reads and writes. */
+export const TRACKING_CHANNEL_FIELDS = {
+    face: 'faceSourceId',
+    pose: 'poseSourceId',
+    hands: 'handSourceId',
+};
+export const HAND_TRACKING_MODES = ['arms', 'fingers'];
+export function isHandTrackingMode(v) {
+    return HAND_TRACKING_MODES.includes(v);
+}
+export const DEFAULT_MEDIAPIPE_CONFIG = {
+    deviceId: '',
+    mirror: true,
+    face: true,
+    hands: true,
+    body: false,
+    delegate: 'CPU',
+};
+/** Whether a source kind uses a network face receiver. */
 export function isFaceSourceKind(kind) {
     return TRACKING_SOURCE_IDS.includes(kind);
 }
+/** Whether a source uses a network body receiver. */
+export function isPoseSource(source) {
+    return POSE_SOURCE_IDS.includes(source.kind);
+}
+const FACE_ONLY = ['face'];
+const POSE_ONLY = ['pose'];
+/** The channels a kind can supply; the webcam's are further gated by its task switches. */
+export function sourceKindChannels(kind) {
+    return kind === 'mediapipe' ? TRACKING_CHANNELS : isFaceSourceKind(kind) ? FACE_ONLY : POSE_ONLY;
+}
+/** Whether this source can supply a channel; {@link sourceChannelEnabled} folds in the switches. */
+export function sourceSupportsChannel(source, channel) {
+    if (!sourceKindChannels(source.kind).includes(channel))
+        return false;
+    if (source.kind !== 'mediapipe')
+        return true;
+    const config = source.mediapipe ?? DEFAULT_MEDIAPIPE_CONFIG;
+    return channel === 'pose' ? config.body : config[channel];
+}
+/**
+ * Whether a source feeds a channel right now: its own switch, its task, and — for a network
+ * face or body source — the channel master. Webcam tasks and hands answer to the source switch alone.
+ */
+export function sourceChannelEnabled(source, channel, masters) {
+    if (!source.enabled || !sourceSupportsChannel(source, channel))
+        return false;
+    return channel === 'hands' || source.kind === 'mediapipe' || masters[channel];
+}
+/**
+ * Whether another source already holds `port` and would contend for it. iFacialMocap instances
+ * share one socket per port (frames demux by phone); every other pairing EADDRINUSEs a listener.
+ */
+export function trackingPortTaken(sources, kind, port, excludeId) {
+    return sources.some(s => s.id !== excludeId && s.port === port && !(s.kind === 'ifacialmocap' && kind === 'ifacialmocap'));
+}
 export const EFFECTS_QUALITY_LEVELS = ['low', 'medium', 'high'];
 /**
  * Accepted values for `performance.fpsLimit`; 0 = unlimited. Anything else is snapped
@@ -79,14 +144,20 @@ const VTS_INPUT_NAMES = [
     'TongueOut',
 ];
 /**
- * The valid `id`s for `input` inject targets, and the names a model's `.vtube.json`
- * references: VTS's derived vocabulary plus every raw ARKit channel.
+ * Default input vocabulary: face and hand inputs, raw ARKit channels and controller profile 1.
+ * Additional controller profile ids are accepted by isInputName without appearing in this list.
  */
-export const INPUT_NAMES = [...VTS_INPUT_NAMES, ...ARKIT_INPUT_NAMES];
+export const INPUT_NAMES = [
+    ...VTS_INPUT_NAMES,
+    ...VOICE_INPUT_NAMES,
+    ...HAND_INPUT_NAMES,
+    ...ARKIT_INPUT_NAMES,
+    ...BASE_CONTROLLER_INPUT_NAMES,
+];
 const INPUT_NAME_SET = new Set(INPUT_NAMES);
 /** Whether an untrusted string names a tracking input — the guard every wire boundary needs. */
 export function isInputName(v) {
-    return INPUT_NAME_SET.has(v);
+    return INPUT_NAME_SET.has(v) || isControllerInputName(v);
 }
 /**
  * VTS-vocabulary inputs that are the same float as one raw ARKit channel (the desktop derives
@@ -145,12 +216,19 @@ const VTS_INPUT_RANGES = {
 /** ARKit blendshapes are unit-scale by definition, so every raw channel shares one span. */
 const ARKIT_RANGE = [0, 1];
 /**
- * Each input's natural span — the units a sender should write, and the input range a new
- * binding starts from. Head angles are **degrees**; the rest are unitless.
+ * Default inputs' natural spans. Use getInputRange for a dynamically numbered controller input.
+ * Head angles are degrees; the rest are unitless.
  */
 export const INPUT_RANGES = {
     ...VTS_INPUT_RANGES,
+    ...VOICE_INPUT_RANGES,
+    ...HAND_INPUT_RANGES,
+    ...BASE_CONTROLLER_INPUT_RANGES,
     // `fromEntries` widens the key type back to `string`; the annotation above is what keeps
     // this exhaustive, and it fails to typecheck if a name ever lacks a range.
     ...Object.fromEntries(ARKIT_INPUT_NAMES.map(n => [n, ARKIT_RANGE])),
 };
+/** Natural span of any valid input, including dynamically numbered controller profiles. */
+export function getInputRange(name) {
+    return isControllerInputName(name) ? controllerInputRange(name) : INPUT_RANGES[name];
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@laplace.live/persona-sdk",
-  "version": "0.18.1",
+  "version": "1.1.0",
   "description": "TypeScript SDK and wire schema for the LAPLACE Persona plugin API",
   "license": "MIT",
   "type": "module",
@@ -16,10 +16,6 @@
     ".": {
       "types": "./dist/index.d.ts",
       "import": "./dist/index.js"
-    },
-    "./effects": {
-      "types": "./dist/effects.d.ts",
-      "import": "./dist/effects.js"
     }
   },
   "files": [
@@ -30,22 +26,12 @@
     "provenance": true
   },
   "devDependencies": {
-    "@types/three": "^0.185.4",
     "rimraf": "^6.1.3",
-    "three": "0.185.1",
     "typescript": "~6.0.3",
-    "vitest": "^4.1.11"
-  },
-  "peerDependencies": {
-    "@types/three": "^0.185.0"
-  },
-  "peerDependenciesMeta": {
-    "@types/three": {
-      "optional": true
-    }
+    "vitest": "^5.0.0"
   },
   "engines": {
-    "node": ">=22"
+    "node": ">=24"
   },
   "dependencies": {
     "zod": "^4.5.4"

package/dist/effects.d.ts

@@ -1,79 +0,0 @@
-import type * as TSL from 'three/tsl';
-import type { Color, Node, Vector2, Vector3, Vector4 } from 'three/webgpu';
-export * from './values/custom-effect.ts';
-/** A vec4-typed TSL node with its swizzles — what `sample` returns and `build` composes over. */
-export type CustomEffectVec4 = ReturnType<typeof TSL.nodeObject<Node<'vec4'>>>;
-/** A float-typed TSL node with its operators — what `viewZ` returns. */
-export type CustomEffectFloat = ReturnType<typeof TSL.nodeObject<Node<'float'>>>;
-/** A vec3-typed TSL node with its swizzles — what the camera probes return. */
-export type CustomEffectVec3 = ReturnType<typeof TSL.nodeObject<Node<'vec3'>>>;
-/** What an author's module receives. Not a security boundary — just the useful surface. */
-export interface CustomEffectApi {
-    /** The whole `three/tsl` namespace: `vec4`, `uv`, `uniform`, `mix`, `Fn`, … */
-    tsl: typeof TSL;
-    three: {
-        Color: typeof Color;
-        Vector2: typeof Vector2;
-        Vector3: typeof Vector3;
-        Vector4: typeof Vector4;
-    };
-    /**
-     * Stock three display nodes an effect cannot build for itself. `afterImage`
-     * is the seam for anything temporal: correct frame feedback needs two render
-     * targets ping-ponged plus a renderer reference, and `build()` hands authors
-     * neither. Upstream types return the bare class, hiding the TSL swizzles the
-     * runtime proxy carries; this signature types the echo as what authors use it
-     * as — a sampleable vec4 image.
-     */
-    nodes: {
-        afterImage: (node: Node, damp?: Node | number) => CustomEffectVec4;
-    };
-}
-/** Build-time context for one custom effect. */
-export interface CustomEffectBuildContext {
-    /** Sample the incoming frame at any UV — the seam that makes warps and blurs possible. */
-    sample: (at: unknown) => CustomEffectVec4;
-    /** Screen UV of the pixel being shaded. */
-    uv: typeof TSL.uv;
-    /**
-     * View-space Z of the 3D scene at this pixel — three's convention: 0 at the
-     * camera, more negative with distance, the far plane where nothing was drawn.
-     * The Live2D layer and 2D objects contribute no depth. Costs one depth read,
-     * and only if called.
-     */
-    viewZ: () => CustomEffectFloat;
-    /**
-     * Normalized world-space direction of the scene camera's view ray through
-     * this pixel, fov and aspect folded in. Inside the post pass TSL's own
-     * `cameraPosition`/`cameraWorldMatrix` describe the fullscreen quad's
-     * camera — this is the real one. Anchor a field here instead of `uv()` and
-     * it holds still in the world while the camera orbits.
-     */
-    worldRay: () => CustomEffectVec3;
-    /** The scene camera's world position in metres, as a per-frame uniform. */
-    cameraPosition: () => CustomEffectVec3;
-    /** Hand a build-time disposable (a blur render target) to the chain's transient list. */
-    track: (t: {
-        dispose?: () => void;
-    }) => void;
-}
-/** What an author's module returns. Everything but `build` is optional. */
-export interface CustomEffectModule {
-    /** Uniform nodes keyed by manifest param name; param edits write straight into `.value`. */
-    uniforms?: Record<string, {
-        value: unknown;
-    }>;
-    /** Compose the effect over `input` — the frame so far, already sampleable — and return the result node. */
-    build: (input: Node, ctx: CustomEffectBuildContext) => Node;
-    /** Per-frame CPU state. `dt` and `elapsed` are seconds. */
-    update?: (dt: number, elapsed: number) => void;
-    dispose?: () => void;
-}
-/** The module's default export. Runs once per load — mint uniforms here so they survive rebuilds. */
-export type CustomEffectFactory = (api: CustomEffectApi) => CustomEffectModule;
-/** Why an effect is not rendering, for the panel's error row. */
-export interface CustomEffectFault {
-    slug: string;
-    phase: 'load' | 'build' | 'update';
-    message: string;
-}

package/dist/effects.js

@@ -1,6 +0,0 @@
-// The custom-effect authoring contract: what an effect module receives, what it
-// returns, and (re-exported) the manifest shape naming its params. Everything
-// touching three is type-only — the desktop hands the runtime to the module's
-// factory, because a module loaded from `persona://` can import nothing.
-// Typechecking against this entry needs `@types/three` (optional peer).
-export * from "./values/custom-effect.js";

package/dist/values/custom-effect.d.ts

@@ -1,44 +0,0 @@
-/** How wide a manifest may open a slider; keeps a typo'd range from making one unusable. */
-export declare const CUSTOM_EFFECT_PARAM_LIMIT = 1000000;
-/** Enabled custom effects one scene may compose — each costs a full-frame RTT. Disabled entries are uncapped. */
-export declare const CUSTOM_EFFECTS_MAX = 8;
-/** Params per effect. Past this the panel section stops being navigable. */
-export declare const CUSTOM_EFFECT_PARAM_MAX = 32;
-export type CustomEffectParamKind = 'number' | 'boolean' | 'color';
-/** One author-declared control. `kind` picks the panel widget and the healing rule. */
-export interface CustomEffectParam {
-    kind: CustomEffectParamKind;
-    /** Slider/field label. Author-supplied, so never translated. */
-    label: string;
-    /** Numbers: the clamp range and readout. Booleans and colors ignore these. */
-    default: number | boolean | string;
-    min?: number;
-    max?: number;
-    step?: number;
-    digits?: number;
-    unit?: string;
-}
-/** A validated `manifest.json`. */
-export interface CustomEffectManifest {
-    name: string;
-    /** Author-declared, shown in the panel's detail row. */
-    version?: string;
-    author?: string;
-    description?: string;
-    params: Record<string, CustomEffectParam>;
-}
-export declare function isCustomEffectSlug(v: unknown): v is string;
-/**
- * Validate a parsed `manifest.json`. Null when it carries no usable name —
- * everything else degrades (a bad param is dropped, not fatal), because a
- * half-typed manifest should still show the author what already works.
- */
-export declare function parseCustomEffectManifest(raw: unknown): CustomEffectManifest | null;
-/** Every declared param at its default — the params half of a fresh scene entry. */
-export declare function defaultCustomEffectParams(manifest: CustomEffectManifest): Record<string, number | boolean | string>;
-/**
- * Clamp saved params to the manifest that is installed now. Unknown keys are
- * kept: an author mid-edit who renames a param back should not find the value
- * gone, and a param costs nothing until the module reads it.
- */
-export declare function healCustomEffectParams(raw: unknown, manifest: CustomEffectManifest | null): Record<string, number | boolean | string>;

package/dist/values/custom-effect.js

@@ -1,136 +0,0 @@
-// User-authored effects: the manifest contract and its validator.
-//
-// An effect folder holds `manifest.json` (plain data — this file's shape) and
-// `effect.js` (a TSL builder the stage worker evaluates). The split is what lets
-// main heal a scene's saved params without ever running author code: only the
-// worker imports the module, and it is the sole place third-party JS runs.
-//
-// Params mirror EffectParamSpec so the panel renders both kinds of effect with
-// the same sliders — but these arrive at runtime from disk, so everything here
-// validates rather than trusting the type.
-import { finiteOr, isFiniteNumber, isRecord, nonEmptyString } from "./guards.js";
-import { hexColorOr } from "./limits.js";
-/** How wide a manifest may open a slider; keeps a typo'd range from making one unusable. */
-export const CUSTOM_EFFECT_PARAM_LIMIT = 1e6;
-/** Enabled custom effects one scene may compose — each costs a full-frame RTT. Disabled entries are uncapped. */
-export const CUSTOM_EFFECTS_MAX = 8;
-/** Params per effect. Past this the panel section stops being navigable. */
-export const CUSTOM_EFFECT_PARAM_MAX = 32;
-/** Folder-name rule: lowercase, dash-separated. Also the on-disk path segment, so no dots or slashes. */
-const SLUG_RE = /^[a-z0-9][a-z0-9-]{0,63}$/;
-export function isCustomEffectSlug(v) {
-    return typeof v === 'string' && SLUG_RE.test(v);
-}
-function bounded(v) {
-    return Math.min(CUSTOM_EFFECT_PARAM_LIMIT, Math.max(-CUSTOM_EFFECT_PARAM_LIMIT, v));
-}
-function parseParam(raw) {
-    if (!isRecord(raw))
-        return null;
-    const label = nonEmptyString(raw.label);
-    if (label === null)
-        return null;
-    if (raw.kind === 'boolean')
-        return { kind: 'boolean', label, default: raw.default === true };
-    if (raw.kind === 'color')
-        return { kind: 'color', label, default: hexColorOr(raw.default, '#ffffff') };
-    // Numbers are the default kind: an omitted or unknown `kind` is far more often
-    // a slider the author forgot to tag than a control they meant to drop.
-    // Both ends clamp into ±LIMIT before ordering, or a range wholly past the
-    // limit would keep one end beyond it.
-    const lo = bounded(finiteOr(raw.min, 0));
-    const hi = bounded(finiteOr(raw.max, 1));
-    // An inverted or empty range would leave a slider that cannot move; widen to the default's own value.
-    const min = Math.min(lo, hi);
-    const max = Math.max(lo, hi);
-    const def = Math.min(max, Math.max(min, finiteOr(raw.default, min)));
-    const step = Math.min(max - min || 1, Math.abs(finiteOr(raw.step, 0.01)) || 0.01);
-    return {
-        kind: 'number',
-        label,
-        default: def,
-        min,
-        max,
-        step,
-        ...(isFiniteNumber(raw.digits) && { digits: Math.min(6, Math.max(0, Math.trunc(raw.digits))) }),
-        ...(typeof raw.unit === 'string' && raw.unit !== '' && { unit: raw.unit.slice(0, 8) }),
-    };
-}
-/**
- * Validate a parsed `manifest.json`. Null when it carries no usable name —
- * everything else degrades (a bad param is dropped, not fatal), because a
- * half-typed manifest should still show the author what already works.
- */
-export function parseCustomEffectManifest(raw) {
-    if (!isRecord(raw))
-        return null;
-    const name = nonEmptyString(raw.name);
-    if (name === null)
-        return null;
-    const params = {};
-    if (isRecord(raw.params)) {
-        for (const [key, value] of Object.entries(raw.params)) {
-            if (Object.keys(params).length >= CUSTOM_EFFECT_PARAM_MAX)
-                break;
-            // The key is a JS identifier on the uniforms object the module builds; a
-            // key it cannot name would silently never receive its value.
-            if (!/^[A-Za-z_$][\w$]*$/.test(key))
-                continue;
-            const param = parseParam(value);
-            if (param)
-                params[key] = param;
-        }
-    }
-    return {
-        name: name.slice(0, 64),
-        params,
-        ...(typeof raw.version === 'string' && raw.version !== '' && { version: raw.version.slice(0, 32) }),
-        ...(typeof raw.author === 'string' && raw.author !== '' && { author: raw.author.slice(0, 64) }),
-        ...(typeof raw.description === 'string' &&
-            raw.description !== '' && { description: raw.description.slice(0, 280) }),
-    };
-}
-/** Every declared param at its default — the params half of a fresh scene entry. */
-export function defaultCustomEffectParams(manifest) {
-    const out = {};
-    for (const [key, param] of Object.entries(manifest.params))
-        out[key] = param.default;
-    return out;
-}
-/** A primitive a saved param may hold; non-finite numbers are junk, not tuning. */
-function isParamValue(v) {
-    return isFiniteNumber(v) || typeof v === 'boolean' || typeof v === 'string';
-}
-/**
- * Clamp saved params to the manifest that is installed now. Unknown keys are
- * kept: an author mid-edit who renames a param back should not find the value
- * gone, and a param costs nothing until the module reads it.
- */
-export function healCustomEffectParams(raw, manifest) {
-    const src = isRecord(raw) ? raw : {};
-    const out = {};
-    for (const [key, value] of Object.entries(src)) {
-        const spec = manifest?.params[key];
-        if (!spec) {
-            // No spec to clamp against — an unknown key, or no manifest at all (effect
-            // not installed on this machine); keeping primitives is what lets a
-            // reinstall restore the user's tuning.
-            if (isParamValue(value))
-                out[key] = value;
-            continue;
-        }
-        if (spec.kind === 'boolean')
-            out[key] = typeof value === 'boolean' ? value : spec.default === true;
-        else if (spec.kind === 'color')
-            out[key] = hexColorOr(value, String(spec.default));
-        else {
-            const lo = spec.min ?? 0;
-            const hi = spec.max ?? 1;
-            out[key] = Math.min(hi, Math.max(lo, finiteOr(value, Number(spec.default))));
-        }
-    }
-    if (manifest)
-        for (const [key, spec] of Object.entries(manifest.params))
-            out[key] ??= spec.default;
-    return out;
-}