@viji-dev/core 0.3.25 → 0.3.27

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (19) hide show
  1. package/README.md +1 -1
  2. package/dist/artist-dts-p5.js +1 -1
  3. package/dist/artist-dts.js +1 -1
  4. package/dist/artist-global.d.ts +6 -43
  5. package/dist/artist-jsdoc.d.ts +3 -1
  6. package/dist/assets/cv-tasks.worker.js +108 -32
  7. package/dist/assets/{viji.worker-Zg128woJ.js → viji.worker-jTmB7qoQ.js} +546 -209
  8. package/dist/assets/viji.worker-jTmB7qoQ.js.map +1 -0
  9. package/dist/docs-api.js +2145 -513
  10. package/dist/{essentia-wasm.web-Dq7KVAx1.js → essentia-wasm.web-DtvIO6D8.js} +2 -2
  11. package/dist/{essentia-wasm.web-Dq7KVAx1.js.map → essentia-wasm.web-DtvIO6D8.js.map} +1 -1
  12. package/dist/{index-C6fMmKQj.js → index-9YDi9UBP.js} +1439 -1200
  13. package/dist/index-9YDi9UBP.js.map +1 -0
  14. package/dist/index.d.ts +153 -133
  15. package/dist/index.js +1 -1
  16. package/dist/shader-uniforms.js +540 -60
  17. package/package.json +1 -1
  18. package/dist/assets/viji.worker-Zg128woJ.js.map +0 -1
  19. package/dist/index-C6fMmKQj.js.map +0 -1
package/README.md CHANGED
@@ -1143,7 +1143,7 @@ function render(viji, p5) {
1143
1143
  - ✅ All P5.js drawing functions (shapes, colors, transforms, typography)
1144
1144
  - ✅ P5.js math utilities (`noise()`, `random()`, `map()`, `lerp()`)
1145
1145
  - ✅ P5.js vectors (`p5.Vector` class)
1146
- - ✅ WebGL mode (`p5.WEBGL`)
1146
+ - ✅ WEBGL main canvas (`// @renderer p5 webgl` — Viji creates the canvas; do not call `createCanvas`)
1147
1147
  - ✅ Full Viji integration (audio, video, parameters, interaction)
1148
1148
 
1149
1149
  ### What Doesn't Work
package/dist/artist-dts-p5.js CHANGED
@@ -1 +1 @@
1
- export const artistDtsP5 = "// Viji Artist API - Global Type Definitions\n// All types are placed inside declare global {} because this file uses export {}\n// for top-level await support, which makes it a module (where top-level declarations\n// would otherwise be module-scoped, not global).\n\ndeclare global {\n interface AudioAPI {\r\n isConnected: boolean;\r\n volume: {\r\n current: number;\r\n peak: number;\r\n smoothed: number;\r\n };\r\n bands: {\r\n low: number;\r\n lowMid: number;\r\n mid: number;\r\n highMid: number;\r\n high: number;\r\n lowSmoothed: number;\r\n lowMidSmoothed: number;\r\n midSmoothed: number;\r\n highMidSmoothed: number;\r\n highSmoothed: number;\r\n };\r\n beat: {\r\n kick: number;\r\n snare: number;\r\n hat: number;\r\n any: number;\r\n kickSmoothed: number;\r\n snareSmoothed: number;\r\n hatSmoothed: number;\r\n anySmoothed: number;\r\n triggers: {\r\n any: boolean;\r\n kick: boolean;\r\n snare: boolean;\r\n hat: boolean;\r\n };\r\n events: Array<{\r\n type: 'kick' | 'snare' | 'hat';\r\n time: number;\r\n strength: number;\r\n }>;\r\n bpm: number;\r\n confidence: number;\r\n isLocked: boolean;\r\n };\r\n spectral: {\r\n brightness: number;\r\n flatness: number;\r\n };\r\n getFrequencyData: () => Uint8Array;\r\n getWaveform: () => Float32Array;\r\n }\r\n\n interface ButtonConfig {\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface ButtonParameter {\r\n value: boolean;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n interface CaptureFrameOptions {\r\n /** Output format: 'blob' for encoded image, 'bitmap' for GPU-friendly ImageBitmap */\r\n format?: 'blob' | 'bitmap';\r\n /** MIME type for blob output (ignored for bitmap), e.g., 'image/png', 'image/jpeg', 'image/webp' */\r\n type?: string;\r\n /**\r\n * Target resolution.\r\n * - number: scale factor relative to current canvas size (e.g., 0.5 = 50%)\r\n * - { width, height }: exact output size; if aspect ratio differs from canvas,\r\n * the source is center-cropped to match the target aspect ratio before scaling\r\n */\r\n resolution?: number | {\r\n width: number;\r\n height: number;\r\n };\r\n }\r\n\n interface ColorConfig {\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface ColorParameter {\r\n value: string;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n type CVFeature = 'faceDetection' | 'faceMesh' | 'handTracking' | 'poseDetection' | 'bodySegmentation' | 'emotionDetection';\r\n\n type CVFrameRateMode = 'full' | 'half' | 'quarter' | 'eighth';\r\n\n interface DeviceMotionData {\r\n /** Acceleration without gravity (m/s²) */\r\n acceleration: {\r\n x: number | null;\r\n y: number | null;\r\n z: number | null;\r\n } | null;\r\n /** Acceleration including gravity (m/s²) */\r\n accelerationIncludingGravity: {\r\n x: number | null;\r\n y: number | null;\r\n z: number | null;\r\n } | null;\r\n /** Rotation rate (degrees/second) */\r\n rotationRate: {\r\n alpha: number | null;\r\n beta: number | null;\r\n gamma: number | null;\r\n } | null;\r\n /** Interval between updates (milliseconds) */\r\n interval: number;\r\n }\r\n\n interface DeviceOrientationData {\r\n /** Rotation around Z-axis (0-360 degrees, compass heading) */\r\n alpha: number | null;\r\n /** Rotation around X-axis (-180 to 180 degrees, front-to-back tilt) */\r\n beta: number | null;\r\n /** Rotation around Y-axis (-90 to 90 degrees, left-to-right tilt) */\r\n gamma: number | null;\r\n /** True if using magnetometer (compass) for absolute orientation */\r\n absolute: boolean;\r\n }\r\n\n interface DeviceSensorState {\r\n motion: DeviceMotionData | null;\r\n orientation: DeviceOrientationData | null;\r\n }\r\n\n interface DeviceState extends DeviceSensorState {\r\n /** Unique device identifier */\r\n id: string;\r\n /** User-friendly device name */\r\n name: string;\r\n /** Device camera video (null if not available) */\r\n video: VideoAPI | null;\r\n }\r\n\n interface FaceBlendshapes {\r\n browDownLeft: number;\r\n browDownRight: number;\r\n browInnerUp: number;\r\n browOuterUpLeft: number;\r\n browOuterUpRight: number;\r\n cheekPuff: number;\r\n cheekSquintLeft: number;\r\n cheekSquintRight: number;\r\n eyeBlinkLeft: number;\r\n eyeBlinkRight: number;\r\n eyeLookDownLeft: number;\r\n eyeLookDownRight: number;\r\n eyeLookInLeft: number;\r\n eyeLookInRight: number;\r\n eyeLookOutLeft: number;\r\n eyeLookOutRight: number;\r\n eyeLookUpLeft: number;\r\n eyeLookUpRight: number;\r\n eyeSquintLeft: number;\r\n eyeSquintRight: number;\r\n eyeWideLeft: number;\r\n eyeWideRight: number;\r\n jawForward: number;\r\n jawLeft: number;\r\n jawOpen: number;\r\n jawRight: number;\r\n mouthClose: number;\r\n mouthDimpleLeft: number;\r\n mouthDimpleRight: number;\r\n mouthFrownLeft: number;\r\n mouthFrownRight: number;\r\n mouthFunnel: number;\r\n mouthLeft: number;\r\n mouthLowerDownLeft: number;\r\n mouthLowerDownRight: number;\r\n mouthPressLeft: number;\r\n mouthPressRight: number;\r\n mouthPucker: number;\r\n mouthRight: number;\r\n mouthRollLower: number;\r\n mouthRollUpper: number;\r\n mouthShrugLower: number;\r\n mouthShrugUpper: number;\r\n mouthSmileLeft: number;\r\n mouthSmileRight: number;\r\n mouthStretchLeft: number;\r\n mouthStretchRight: number;\r\n mouthUpperUpLeft: number;\r\n mouthUpperUpRight: number;\r\n noseSneerLeft: number;\r\n noseSneerRight: number;\r\n tongueOut: number;\r\n }\r\n\n interface FaceData {\r\n id: number;\r\n bounds: {\r\n x: number;\r\n y: number;\r\n width: number;\r\n height: number;\r\n };\r\n center: {\r\n x: number;\r\n y: number;\r\n };\r\n confidence: number;\r\n landmarks: {\r\n x: number;\r\n y: number;\r\n z?: number;\r\n }[];\r\n expressions: {\r\n neutral: number;\r\n happy: number;\r\n sad: number;\r\n angry: number;\r\n surprised: number;\r\n disgusted: number;\r\n fearful: number;\r\n };\r\n headPose: {\r\n pitch: number;\r\n yaw: number;\r\n roll: number;\r\n };\r\n blendshapes: FaceBlendshapes;\r\n }\r\n\n type FrameRateMode = 'full' | 'half';\r\n\n interface FrequencyBand {\r\n name: string;\r\n min: number;\r\n max: number;\r\n }\r\n\n interface HandData {\r\n id: number;\r\n handedness: 'left' | 'right';\r\n confidence: number;\r\n bounds: {\r\n x: number;\r\n y: number;\r\n width: number;\r\n height: number;\r\n };\r\n landmarks: {\r\n x: number;\r\n y: number;\r\n z: number;\r\n }[];\r\n palm: {\r\n x: number;\r\n y: number;\r\n z: number;\r\n };\r\n fingers: {\r\n thumb: {\r\n tip: {\r\n x: number;\r\n y: number;\r\n z: number;\r\n };\r\n extended: boolean;\r\n };\r\n index: {\r\n tip: {\r\n x: number;\r\n y: number;\r\n z: number;\r\n };\r\n extended: boolean;\r\n };\r\n middle: {\r\n tip: {\r\n x: number;\r\n y: number;\r\n z: number;\r\n };\r\n extended: boolean;\r\n };\r\n ring: {\r\n tip: {\r\n x: number;\r\n y: number;\r\n z: number;\r\n };\r\n extended: boolean;\r\n };\r\n pinky: {\r\n tip: {\r\n x: number;\r\n y: number;\r\n z: number;\r\n };\r\n extended: boolean;\r\n };\r\n };\r\n gestures: {\r\n fist: number;\r\n openPalm: number;\r\n peace: number;\r\n thumbsUp: number;\r\n pointing: number;\r\n };\r\n }\r\n\n interface ImageConfig {\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface ImageParameter {\r\n value: ImageBitmap | null;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n interface KeyboardAPI {\r\n isPressed(key: string): boolean;\r\n wasPressed(key: string): boolean;\r\n wasReleased(key: string): boolean;\r\n activeKeys: Set<string>;\r\n pressedThisFrame: Set<string>;\r\n releasedThisFrame: Set<string>;\r\n lastKeyPressed: string;\r\n lastKeyReleased: string;\r\n shift: boolean;\r\n ctrl: boolean;\r\n alt: boolean;\r\n meta: boolean;\r\n textureData: Uint8Array;\r\n }\r\n\n interface MouseAPI {\r\n x: number;\r\n y: number;\r\n isInCanvas: boolean;\r\n isPressed: boolean;\r\n leftButton: boolean;\r\n rightButton: boolean;\r\n middleButton: boolean;\r\n deltaX: number;\r\n deltaY: number;\r\n wheelDelta: number;\r\n wheelX: number;\r\n wheelY: number;\r\n wasPressed: boolean;\r\n wasReleased: boolean;\r\n wasMoved: boolean;\r\n }\r\n\n interface NumberConfig {\r\n min?: number;\r\n max?: number;\r\n step?: number;\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface NumberParameter {\r\n value: number;\r\n min: number;\r\n max: number;\r\n step: number;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n type ParameterCategory = 'audio' | 'video' | 'interaction' | 'general';\r\n\n interface PointerAPI {\r\n x: number;\r\n y: number;\r\n deltaX: number;\r\n deltaY: number;\r\n isDown: boolean;\r\n wasPressed: boolean;\r\n wasReleased: boolean;\r\n isInCanvas: boolean;\r\n type: 'mouse' | 'touch' | 'none';\r\n }\r\n\n interface PoseData {\r\n confidence: number;\r\n landmarks: {\r\n x: number;\r\n y: number;\r\n z: number;\r\n visibility: number;\r\n /**\n * P5-compatible image object (lazy-loaded, auto-converted from .value)\n * Only available in P5 renderer mode. Use with p5.image() function.\n * Example: p5.image(myImage.p5, x, y)\n */\n p5?: {\n canvas: OffscreenCanvas; // P5.js looks for img.canvas || img.elt\n elt: OffscreenCanvas; // Fallback for compatibility\n width: number; // Logical width\n height: number; // Logical height\n };\n }[];\r\n face: {\r\n x: number;\r\n y: number;\r\n }[];\r\n torso: {\r\n x: number;\r\n y: number;\r\n }[];\r\n leftArm: {\r\n x: number;\r\n y: number;\r\n }[];\r\n rightArm: {\r\n x: number;\r\n y: number;\r\n }[];\r\n leftLeg: {\r\n x: number;\r\n y: number;\r\n }[];\r\n rightLeg: {\r\n x: number;\r\n y: number;\r\n }[];\r\n }\r\n\n type Resolution = {\r\n width: number;\r\n height: number;\r\n };\r\n\n interface SegmentationData {\r\n mask: Uint8Array;\r\n width: number;\r\n height: number;\r\n }\r\n\n interface SelectConfig {\r\n options: string[] | number[];\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface SelectParameter {\r\n value: string | number;\r\n options: string[] | number[];\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n interface SliderConfig {\r\n min?: number;\r\n max?: number;\r\n step?: number;\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface SliderParameter {\r\n value: number;\r\n min: number;\r\n max: number;\r\n step: number;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n interface TextConfig {\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n maxLength?: number;\r\n }\r\n\n interface TextParameter {\r\n value: string;\r\n maxLength?: number;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n interface ToggleConfig {\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface ToggleParameter {\r\n value: boolean;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n interface TouchAPI {\r\n points: TouchPoint[];\r\n count: number;\r\n started: TouchPoint[];\r\n moved: TouchPoint[];\r\n ended: TouchPoint[];\r\n primary: TouchPoint | null;\r\n }\r\n\n interface TouchPoint {\r\n id: number;\r\n x: number;\r\n y: number;\r\n pressure: number;\r\n radius: number;\r\n radiusX: number;\r\n radiusY: number;\r\n rotationAngle: number;\r\n force: number;\r\n isInCanvas: boolean;\r\n deltaX: number;\r\n deltaY: number;\r\n velocity: {\r\n x: number;\r\n y: number;\r\n };\r\n isNew: boolean;\r\n isActive: boolean;\r\n isEnding: boolean;\r\n }\r\n\n const VERSION = \"0.2.20\";\r\n\n interface VideoAPI {\r\n isConnected: boolean;\r\n currentFrame: OffscreenCanvas | ImageBitmap | null;\r\n frameWidth: number;\r\n frameHeight: number;\r\n frameRate: number;\r\n getFrameData: () => ImageData | null;\r\n faces: FaceData[];\r\n hands: HandData[];\r\n pose: PoseData | null;\r\n segmentation: SegmentationData | null;\r\n cv: {\r\n enableFaceDetection(enabled: boolean): Promise<void>;\r\n enableFaceMesh(enabled: boolean): Promise<void>;\r\n enableEmotionDetection(enabled: boolean): Promise<void>;\r\n enableHandTracking(enabled: boolean): Promise<void>;\r\n enablePoseDetection(enabled: boolean): Promise<void>;\r\n enableBodySegmentation(enabled: boolean): Promise<void>;\r\n getActiveFeatures(): CVFeature[];\r\n isProcessing(): boolean;\r\n };\r\n }\r\n\n interface VijiAPI {\r\n canvas: OffscreenCanvas;\r\n ctx?: OffscreenCanvasRenderingContext2D;\r\n gl?: WebGLRenderingContext | WebGL2RenderingContext;\r\n width: number;\r\n height: number;\r\n time: number;\r\n deltaTime: number;\r\n frameCount: number;\r\n fps: number;\r\n audio: AudioAPI;\r\n video: VideoAPI;\r\n streams: VideoAPI[];\r\n mouse: MouseAPI;\r\n keyboard: KeyboardAPI;\r\n touches: TouchAPI;\r\n pointer: PointerAPI;\r\n device: DeviceSensorState;\r\n devices: DeviceState[];\r\n slider: (defaultValue: number, config: SliderConfig) => SliderParameter;\r\n color: (defaultValue: string, config: ColorConfig) => ColorParameter;\r\n toggle: (defaultValue: boolean, config: ToggleConfig) => ToggleParameter;\r\n select: (defaultValue: string | number, config: SelectConfig) => SelectParameter;\r\n text: (defaultValue: string, config: TextConfig) => TextParameter;\r\n number: (defaultValue: number, config: NumberConfig) => NumberParameter;\r\n image: (defaultValue: null, config: ImageConfig) => ImageParameter;\r\n button: (config: ButtonConfig) => ButtonParameter;\r\n useContext(type: '2d'): OffscreenCanvasRenderingContext2D;\r\n useContext(type: 'webgl'): WebGLRenderingContext;\r\n useContext(type: 'webgl2'): WebGL2RenderingContext;\r\n }\r\n\n // Runtime global - the main viji object\n const viji: VijiAPI;\n \n // Function type aliases (artists define their own render/setup functions)\n type Render = (viji: VijiAPI) => void;\n type Setup = (viji: VijiAPI) => void;\n}\n\n// Module marker (enables top-level await in artist code)\nexport {};\n\n\n// ============================================================\n// VIJI + P5.JS HYBRID MODE\n// ============================================================\n// In Viji's P5 renderer, you don't use standard P5 patterns.\n// Instead, you define these Viji-specific functions:\n\n// P5-specific globals (augments the base declare global block)\ndeclare global {\n // P5 instance type for annotations\n type p5 = p5;\n \n // Type aliases for P5 function signatures (artists define their own functions)\n /**\n * P5 render function signature - called every frame\n * @param viji - Viji API (canvas, audio, video, parameters, etc.)\n * @param p5 - P5.js instance with all P5 methods\n */\n type P5Render = (viji: VijiAPI, p5: p5) => void;\n \n /**\n * P5 setup function signature - called once at initialization\n * @param viji - Viji API (canvas, audio, video, parameters, etc.)\n * @param p5 - P5.js instance with all P5 methods\n */\n type P5Setup = (viji: VijiAPI, p5: p5) => void;\n}\n\n// ============================================================\n// IMPORTANT NOTES\n// ============================================================\n// - Don't use: new p5(), preload(), draw(), or P5's setup()\n// - Viji manages the P5 instance lifecycle\n// - Access P5 methods through the 'p5' parameter: p5.rect(), p5.fill(), etc.\n// - Access Viji features through the 'viji' parameter: viji.audio, viji.mouse, etc.\n// - Define parameters at the top level (before setup/render functions)\n\n// ============================================================\n// P5.JS TYPES (namespace and instance methods)\n// ============================================================\n// This file was auto-generated. Please do not edit it.\n/// <reference path=\"./src/accessibility/describe.d.ts\" />\n/// <reference path=\"./src/accessibility/outputs.d.ts\" />\n/// <reference path=\"./src/color/creating_reading.d.ts\" />\n/// <reference path=\"./src/color/setting.d.ts\" />\n/// <reference path=\"./src/core/shape/2d_primitives.d.ts\" />\n/// <reference path=\"./src/core/shape/attributes.d.ts\" />\n/// <reference path=\"./src/core/shape/curves.d.ts\" />\n/// <reference path=\"./src/core/shape/vertex.d.ts\" />\n/// <reference path=\"./src/core/constants.d.ts\" />\n/// <reference path=\"./src/core/environment.d.ts\" />\n/// <reference path=\"./src/core/rendering.d.ts\" />\n/// <reference path=\"./src/core/structure.d.ts\" />\n/// <reference path=\"./src/core/transform.d.ts\" />\n/// <reference path=\"./src/data/local_storage.d.ts\" />\n/// <reference path=\"./src/data/p5.TypedDict.d.ts\" />\n/// <reference path=\"./src/dom/dom.d.ts\" />\n/// <reference path=\"./src/events/acceleration.d.ts\" />\n/// <reference path=\"./src/events/keyboard.d.ts\" />\n/// <reference path=\"./src/events/mouse.d.ts\" />\n/// <reference path=\"./src/events/touch.d.ts\" />\n/// <reference path=\"./src/image/image.d.ts\" />\n/// <reference path=\"./src/image/loading_displaying.d.ts\" />\n/// <reference path=\"./src/image/pixels.d.ts\" />\n/// <reference path=\"./src/io/files.d.ts\" />\n/// <reference path=\"./src/math/calculation.d.ts\" />\n/// <reference path=\"./src/math/math.d.ts\" />\n/// <reference path=\"./src/math/noise.d.ts\" />\n/// <reference path=\"./src/math/random.d.ts\" />\n/// <reference path=\"./src/math/trigonometry.d.ts\" />\n/// <reference path=\"./src/typography/attributes.d.ts\" />\n/// <reference path=\"./src/typography/loading_displaying.d.ts\" />\n/// <reference path=\"./src/utilities/array_functions.d.ts\" />\n/// <reference path=\"./src/utilities/conversion.d.ts\" />\n/// <reference path=\"./src/utilities/string_functions.d.ts\" />\n/// <reference path=\"./src/utilities/time_date.d.ts\" />\n/// <reference path=\"./src/webgl/3d_primitives.d.ts\" />\n/// <reference path=\"./src/webgl/interaction.d.ts\" />\n/// <reference path=\"./src/webgl/light.d.ts\" />\n/// <reference path=\"./src/webgl/loading.d.ts\" />\n/// <reference path=\"./src/webgl/material.d.ts\" />\n/// <reference path=\"./src/webgl/p5.Camera.d.ts\" />\n/// <reference path=\"./src/webgl/p5.RendererGL.Immediate.d.ts\" />\n/// <reference path=\"./src/webgl/p5.RendererGL.d.ts\" />\n/// <reference path=\"./src/color/p5.Color.d.ts\" />\n/// <reference path=\"./src/core/p5.Element.d.ts\" />\n/// <reference path=\"./src/core/p5.Graphics.d.ts\" />\n/// <reference path=\"./src/image/p5.Image.d.ts\" />\n/// <reference path=\"./src/io/p5.Table.d.ts\" />\n/// <reference path=\"./src/io/p5.TableRow.d.ts\" />\n/// <reference path=\"./src/io/p5.XML.d.ts\" />\n/// <reference path=\"./src/math/p5.Vector.d.ts\" />\n/// <reference path=\"./src/typography/p5.Font.d.ts\" />\n/// <reference path=\"./src/webgl/p5.Framebuffer.d.ts\" />\n/// <reference path=\"./src/webgl/p5.Geometry.d.ts\" />\n/// <reference path=\"./src/webgl/p5.Shader.d.ts\" />\n/// <reference path=\"./src/core/p5.Renderer.d.ts\" />\n/// <reference path=\"./literals.d.ts\" />\n/// <reference path=\"./constants.d.ts\" />\n= p5;\n\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\ninterface p5 extends p5.p5InstanceExtensions {}\n\ndeclare namespace p5 {\n type UNKNOWN_P5_CONSTANT = any;\n // eslint-disable-next-line @typescript-eslint/no-empty-interface\n interface p5InstanceExtensions {}\n}\n\n";
1
+ export const artistDtsP5 = "// Viji Artist API - Global Type Definitions\n// All types are placed inside declare global {} because this file uses export {}\n// for top-level await support, which makes it a module (where top-level declarations\n// would otherwise be module-scoped, not global).\n\ndeclare global {\n interface AudioAPI {\r\n isConnected: boolean;\r\n volume: {\r\n current: number;\r\n peak: number;\r\n smoothed: number;\r\n };\r\n bands: {\r\n low: number;\r\n lowMid: number;\r\n mid: number;\r\n highMid: number;\r\n high: number;\r\n lowSmoothed: number;\r\n lowMidSmoothed: number;\r\n midSmoothed: number;\r\n highMidSmoothed: number;\r\n highSmoothed: number;\r\n };\r\n beat: {\r\n kick: number;\r\n snare: number;\r\n hat: number;\r\n any: number;\r\n kickSmoothed: number;\r\n snareSmoothed: number;\r\n hatSmoothed: number;\r\n anySmoothed: number;\r\n triggers: {\r\n any: boolean;\r\n kick: boolean;\r\n snare: boolean;\r\n hat: boolean;\r\n };\r\n events: Array<{\r\n type: 'kick' | 'snare' | 'hat';\r\n time: number;\r\n strength: number;\r\n }>;\r\n bpm: number;\r\n confidence: number;\r\n isLocked: boolean;\r\n };\r\n spectral: {\r\n brightness: number;\r\n flatness: number;\r\n };\r\n getFrequencyData: () => Uint8Array;\r\n getWaveform: () => Float32Array;\r\n }\r\n\n interface ButtonConfig {\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface ButtonParameter {\r\n value: boolean;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n interface CaptureFrameOptions {\r\n /** Output format: 'blob' for encoded image, 'bitmap' for GPU-friendly ImageBitmap */\r\n format?: 'blob' | 'bitmap';\r\n /** MIME type for blob output (ignored for bitmap), e.g., 'image/png', 'image/jpeg', 'image/webp' */\r\n type?: string;\r\n /**\r\n * Target resolution.\r\n * - number: scale factor relative to current canvas size (e.g., 0.5 = 50%)\r\n * - { width, height }: exact output size; if aspect ratio differs from canvas,\r\n * the source is center-cropped to match the target aspect ratio before scaling\r\n */\r\n resolution?: number | {\r\n width: number;\r\n height: number;\r\n };\r\n }\r\n\n interface ColorConfig {\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface ColorParameter {\r\n value: string;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n type CVFeature = 'faceDetection' | 'faceMesh' | 'handTracking' | 'poseDetection' | 'bodySegmentation' | 'emotionDetection';\r\n\n type CVFrameRateMode = 'full' | 'half' | 'quarter' | 'eighth';\r\n\n interface DeviceMotionData {\r\n /** Acceleration without gravity (m/s²) */\r\n acceleration: {\r\n x: number | null;\r\n y: number | null;\r\n z: number | null;\r\n } | null;\r\n /** Acceleration including gravity (m/s²) */\r\n accelerationIncludingGravity: {\r\n x: number | null;\r\n y: number | null;\r\n z: number | null;\r\n } | null;\r\n /** Rotation rate (degrees/second) */\r\n rotationRate: {\r\n alpha: number | null;\r\n beta: number | null;\r\n gamma: number | null;\r\n } | null;\r\n /** Interval between updates (milliseconds) */\r\n interval: number;\r\n }\r\n\n interface DeviceOrientationData {\r\n /** Rotation around Z-axis (0-360 degrees, compass heading) */\r\n alpha: number | null;\r\n /** Rotation around X-axis (-180 to 180 degrees, front-to-back tilt) */\r\n beta: number | null;\r\n /** Rotation around Y-axis (-90 to 90 degrees, left-to-right tilt) */\r\n gamma: number | null;\r\n /** True if using magnetometer (compass) for absolute orientation */\r\n absolute: boolean;\r\n }\r\n\n interface DeviceSensorState {\r\n motion: DeviceMotionData | null;\r\n orientation: DeviceOrientationData | null;\r\n }\r\n\n interface DeviceState extends DeviceSensorState {\r\n /** Unique device identifier */\r\n id: string;\r\n /** User-friendly device name */\r\n name: string;\r\n /** Device camera video (null if not available) */\r\n video: VideoAPI | null;\r\n /** Device audio stream (null if not available) */\r\n audio: AudioStreamAPI | null;\r\n }\r\n\n interface FaceBlendshapes {\r\n browDownLeft: number;\r\n browDownRight: number;\r\n browInnerUp: number;\r\n browOuterUpLeft: number;\r\n browOuterUpRight: number;\r\n cheekPuff: number;\r\n cheekSquintLeft: number;\r\n cheekSquintRight: number;\r\n eyeBlinkLeft: number;\r\n eyeBlinkRight: number;\r\n eyeLookDownLeft: number;\r\n eyeLookDownRight: number;\r\n eyeLookInLeft: number;\r\n eyeLookInRight: number;\r\n eyeLookOutLeft: number;\r\n eyeLookOutRight: number;\r\n eyeLookUpLeft: number;\r\n eyeLookUpRight: number;\r\n eyeSquintLeft: number;\r\n eyeSquintRight: number;\r\n eyeWideLeft: number;\r\n eyeWideRight: number;\r\n jawForward: number;\r\n jawLeft: number;\r\n jawOpen: number;\r\n jawRight: number;\r\n mouthClose: number;\r\n mouthDimpleLeft: number;\r\n mouthDimpleRight: number;\r\n mouthFrownLeft: number;\r\n mouthFrownRight: number;\r\n mouthFunnel: number;\r\n mouthLeft: number;\r\n mouthLowerDownLeft: number;\r\n mouthLowerDownRight: number;\r\n mouthPressLeft: number;\r\n mouthPressRight: number;\r\n mouthPucker: number;\r\n mouthRight: number;\r\n mouthRollLower: number;\r\n mouthRollUpper: number;\r\n mouthShrugLower: number;\r\n mouthShrugUpper: number;\r\n mouthSmileLeft: number;\r\n mouthSmileRight: number;\r\n mouthStretchLeft: number;\r\n mouthStretchRight: number;\r\n mouthUpperUpLeft: number;\r\n mouthUpperUpRight: number;\r\n noseSneerLeft: number;\r\n noseSneerRight: number;\r\n tongueOut: number;\r\n }\r\n\n interface FaceData {\r\n id: number;\r\n bounds: {\r\n x: number;\r\n y: number;\r\n width: number;\r\n height: number;\r\n };\r\n center: {\r\n x: number;\r\n y: number;\r\n };\r\n confidence: number;\r\n landmarks: {\r\n x: number;\r\n y: number;\r\n z?: number;\r\n }[];\r\n expressions: {\r\n neutral: number;\r\n happy: number;\r\n sad: number;\r\n angry: number;\r\n surprised: number;\r\n disgusted: number;\r\n fearful: number;\r\n };\r\n headPose: {\r\n pitch: number;\r\n yaw: number;\r\n roll: number;\r\n };\r\n blendshapes: FaceBlendshapes;\r\n }\r\n\n type FrameRateMode = 'full' | 'half';\r\n\n interface FrequencyBand {\r\n name: string;\r\n min: number;\r\n max: number;\r\n }\r\n\n interface HandData {\r\n id: number;\r\n handedness: 'left' | 'right';\r\n confidence: number;\r\n bounds: {\r\n x: number;\r\n y: number;\r\n width: number;\r\n height: number;\r\n };\r\n landmarks: {\r\n x: number;\r\n y: number;\r\n z: number;\r\n }[];\r\n palm: {\r\n x: number;\r\n y: number;\r\n z: number;\r\n };\r\n gestures: {\r\n fist: number;\r\n openPalm: number;\r\n peace: number;\r\n thumbsUp: number;\r\n thumbsDown: number;\r\n pointing: number;\r\n iLoveYou: number;\r\n };\r\n }\r\n\n interface ImageConfig {\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface ImageParameter {\r\n value: ImageBitmap | null;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n interface KeyboardAPI {\r\n isPressed(key: string): boolean;\r\n wasPressed(key: string): boolean;\r\n wasReleased(key: string): boolean;\r\n activeKeys: Set<string>;\r\n pressedThisFrame: Set<string>;\r\n releasedThisFrame: Set<string>;\r\n lastKeyPressed: string;\r\n lastKeyReleased: string;\r\n shift: boolean;\r\n ctrl: boolean;\r\n alt: boolean;\r\n meta: boolean;\r\n textureData: Uint8Array;\r\n }\r\n\n interface MouseAPI {\r\n x: number;\r\n y: number;\r\n isInCanvas: boolean;\r\n isPressed: boolean;\r\n leftButton: boolean;\r\n rightButton: boolean;\r\n middleButton: boolean;\r\n deltaX: number;\r\n deltaY: number;\r\n wheelDelta: number;\r\n wheelX: number;\r\n wheelY: number;\r\n wasPressed: boolean;\r\n wasReleased: boolean;\r\n wasMoved: boolean;\r\n }\r\n\n interface NumberConfig {\r\n min?: number;\r\n max?: number;\r\n step?: number;\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface NumberParameter {\r\n value: number;\r\n min: number;\r\n max: number;\r\n step: number;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n type ParameterCategory = 'audio' | 'video' | 'interaction' | 'general';\r\n\n interface PointerAPI {\r\n x: number;\r\n y: number;\r\n deltaX: number;\r\n deltaY: number;\r\n isDown: boolean;\r\n wasPressed: boolean;\r\n wasReleased: boolean;\r\n isInCanvas: boolean;\r\n type: 'mouse' | 'touch' | 'none';\r\n }\r\n\n interface PoseData {\r\n confidence: number;\r\n landmarks: {\r\n x: number;\r\n y: number;\r\n z: number;\r\n visibility: number;\r\n /**\n * P5-compatible image object (lazy-loaded, auto-converted from .value)\n * Only available in P5 renderer mode. Use with p5.image() function.\n * Example: p5.image(myImage.p5, x, y)\n */\n p5?: {\n canvas: OffscreenCanvas; // P5.js looks for img.canvas || img.elt\n elt: OffscreenCanvas; // Fallback for compatibility\n width: number; // Logical width\n height: number; // Logical height\n };\n }[];\r\n face: {\r\n x: number;\r\n y: number;\r\n }[];\r\n torso: {\r\n x: number;\r\n y: number;\r\n }[];\r\n leftArm: {\r\n x: number;\r\n y: number;\r\n }[];\r\n rightArm: {\r\n x: number;\r\n y: number;\r\n }[];\r\n leftLeg: {\r\n x: number;\r\n y: number;\r\n }[];\r\n rightLeg: {\r\n x: number;\r\n y: number;\r\n }[];\r\n }\r\n\n type Resolution = {\r\n width: number;\r\n height: number;\r\n };\r\n\n interface SegmentationData {\r\n mask: Uint8Array;\r\n width: number;\r\n height: number;\r\n }\r\n\n interface SelectConfig {\r\n options: string[] | number[];\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface SelectParameter {\r\n value: string | number;\r\n options: string[] | number[];\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n interface SliderConfig {\r\n min?: number;\r\n max?: number;\r\n step?: number;\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface SliderParameter {\r\n value: number;\r\n min: number;\r\n max: number;\r\n step: number;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n interface TextConfig {\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n maxLength?: number;\r\n }\r\n\n interface TextParameter {\r\n value: string;\r\n maxLength?: number;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n interface ToggleConfig {\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface ToggleParameter {\r\n value: boolean;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n interface TouchAPI {\r\n points: TouchPoint[];\r\n count: number;\r\n started: TouchPoint[];\r\n moved: TouchPoint[];\r\n ended: TouchPoint[];\r\n primary: TouchPoint | null;\r\n }\r\n\n interface TouchPoint {\r\n id: number;\r\n x: number;\r\n y: number;\r\n pressure: number;\r\n radius: number;\r\n radiusX: number;\r\n radiusY: number;\r\n rotationAngle: number;\r\n force: number;\r\n isInCanvas: boolean;\r\n deltaX: number;\r\n deltaY: number;\r\n velocity: {\r\n x: number;\r\n y: number;\r\n };\r\n isNew: boolean;\r\n isActive: boolean;\r\n isEnding: boolean;\r\n }\r\n\n const VERSION = \"0.2.20\";\r\n\n interface VideoAPI {\r\n isConnected: boolean;\r\n currentFrame: OffscreenCanvas | ImageBitmap | null;\r\n frameWidth: number;\r\n frameHeight: number;\r\n frameRate: number;\r\n getFrameData: () => ImageData | null;\r\n faces: FaceData[];\r\n hands: HandData[];\r\n pose: PoseData | null;\r\n segmentation: SegmentationData | null;\r\n cv: {\r\n enableFaceDetection(enabled: boolean): Promise<void>;\r\n enableFaceMesh(enabled: boolean): Promise<void>;\r\n enableEmotionDetection(enabled: boolean): Promise<void>;\r\n enableHandTracking(enabled: boolean): Promise<void>;\r\n enablePoseDetection(enabled: boolean): Promise<void>;\r\n enableBodySegmentation(enabled: boolean): Promise<void>;\r\n getActiveFeatures(): CVFeature[];\r\n isProcessing(): boolean;\r\n };\r\n }\r\n\n interface VijiAPI {\r\n canvas: OffscreenCanvas;\r\n ctx?: OffscreenCanvasRenderingContext2D;\r\n gl?: WebGLRenderingContext | WebGL2RenderingContext;\r\n width: number;\r\n height: number;\r\n time: number;\r\n deltaTime: number;\r\n frameCount: number;\r\n fps: number;\r\n audio: AudioAPI;\r\n video: VideoAPI;\r\n videoStreams: VideoAPI[];\r\n audioStreams: AudioStreamAPI[];\r\n mouse: MouseAPI;\r\n keyboard: KeyboardAPI;\r\n touches: TouchAPI;\r\n pointer: PointerAPI;\r\n device: DeviceSensorState;\r\n devices: DeviceState[];\r\n slider: (defaultValue: number, config: SliderConfig) => SliderParameter;\r\n color: (defaultValue: string, config: ColorConfig) => ColorParameter;\r\n toggle: (defaultValue: boolean, config: ToggleConfig) => ToggleParameter;\r\n select: (defaultValue: string | number, config: SelectConfig) => SelectParameter;\r\n text: (defaultValue: string, config: TextConfig) => TextParameter;\r\n number: (defaultValue: number, config: NumberConfig) => NumberParameter;\r\n image: (defaultValue: null, config: ImageConfig) => ImageParameter;\r\n button: (config: ButtonConfig) => ButtonParameter;\r\n useContext(type: '2d'): OffscreenCanvasRenderingContext2D;\r\n useContext(type: 'webgl'): WebGLRenderingContext;\r\n useContext(type: 'webgl2'): WebGL2RenderingContext;\r\n }\r\n\n // Runtime global - the main viji object\n const viji: VijiAPI;\n \n // Function type aliases (artists define their own render/setup functions)\n type Render = (viji: VijiAPI) => void;\n type Setup = (viji: VijiAPI) => void;\n}\n\n// Module marker (enables top-level await in artist code)\nexport {};\n\n\n// ============================================================\n// VIJI + P5.JS HYBRID MODE\n// ============================================================\n// In Viji's P5 renderer, you don't use standard P5 patterns.\n// Instead, you define these Viji-specific functions:\n\n// P5-specific globals (augments the base declare global block)\ndeclare global {\n // P5 instance type for annotations\n type p5 = p5;\n \n // Type aliases for P5 function signatures (artists define their own functions)\n /**\n * P5 render function signature - called every frame\n * @param viji - Viji API (canvas, audio, video, parameters, etc.)\n * @param p5 - P5.js instance with all P5 methods\n */\n type P5Render = (viji: VijiAPI, p5: p5) => void;\n \n /**\n * P5 setup function signature - called once at initialization\n * @param viji - Viji API (canvas, audio, video, parameters, etc.)\n * @param p5 - P5.js instance with all P5 methods\n */\n type P5Setup = (viji: VijiAPI, p5: p5) => void;\n}\n\n// ============================================================\n// IMPORTANT NOTES\n// ============================================================\n// - Don't use: new p5(), preload(), draw(), or P5's setup()\n// - Viji manages the P5 instance lifecycle (including createCanvas — never call it)\n// - Optional WEBGL: use // @renderer p5 webgl as the first line (default is 2D: // @renderer p5)\n// - Access P5 methods through the 'p5' parameter: p5.rect(), p5.fill(), etc.\n// - Access Viji features through the 'viji' parameter: viji.audio, viji.mouse, etc.\n// - Define parameters at the top level (before setup/render functions)\n\n// ============================================================\n// P5.JS TYPES (namespace and instance methods)\n// ============================================================\n// This file was auto-generated. Please do not edit it.\n/// <reference path=\"./src/accessibility/describe.d.ts\" />\n/// <reference path=\"./src/accessibility/outputs.d.ts\" />\n/// <reference path=\"./src/color/creating_reading.d.ts\" />\n/// <reference path=\"./src/color/setting.d.ts\" />\n/// <reference path=\"./src/core/shape/2d_primitives.d.ts\" />\n/// <reference path=\"./src/core/shape/attributes.d.ts\" />\n/// <reference path=\"./src/core/shape/curves.d.ts\" />\n/// <reference path=\"./src/core/shape/vertex.d.ts\" />\n/// <reference path=\"./src/core/constants.d.ts\" />\n/// <reference path=\"./src/core/environment.d.ts\" />\n/// <reference path=\"./src/core/rendering.d.ts\" />\n/// <reference path=\"./src/core/structure.d.ts\" />\n/// <reference path=\"./src/core/transform.d.ts\" />\n/// <reference path=\"./src/data/local_storage.d.ts\" />\n/// <reference path=\"./src/data/p5.TypedDict.d.ts\" />\n/// <reference path=\"./src/dom/dom.d.ts\" />\n/// <reference path=\"./src/events/acceleration.d.ts\" />\n/// <reference path=\"./src/events/keyboard.d.ts\" />\n/// <reference path=\"./src/events/mouse.d.ts\" />\n/// <reference path=\"./src/events/touch.d.ts\" />\n/// <reference path=\"./src/image/image.d.ts\" />\n/// <reference path=\"./src/image/loading_displaying.d.ts\" />\n/// <reference path=\"./src/image/pixels.d.ts\" />\n/// <reference path=\"./src/io/files.d.ts\" />\n/// <reference path=\"./src/math/calculation.d.ts\" />\n/// <reference path=\"./src/math/math.d.ts\" />\n/// <reference path=\"./src/math/noise.d.ts\" />\n/// <reference path=\"./src/math/random.d.ts\" />\n/// <reference path=\"./src/math/trigonometry.d.ts\" />\n/// <reference path=\"./src/typography/attributes.d.ts\" />\n/// <reference path=\"./src/typography/loading_displaying.d.ts\" />\n/// <reference path=\"./src/utilities/array_functions.d.ts\" />\n/// <reference path=\"./src/utilities/conversion.d.ts\" />\n/// <reference path=\"./src/utilities/string_functions.d.ts\" />\n/// <reference path=\"./src/utilities/time_date.d.ts\" />\n/// <reference path=\"./src/webgl/3d_primitives.d.ts\" />\n/// <reference path=\"./src/webgl/interaction.d.ts\" />\n/// <reference path=\"./src/webgl/light.d.ts\" />\n/// <reference path=\"./src/webgl/loading.d.ts\" />\n/// <reference path=\"./src/webgl/material.d.ts\" />\n/// <reference path=\"./src/webgl/p5.Camera.d.ts\" />\n/// <reference path=\"./src/webgl/p5.RendererGL.Immediate.d.ts\" />\n/// <reference path=\"./src/webgl/p5.RendererGL.d.ts\" />\n/// <reference path=\"./src/color/p5.Color.d.ts\" />\n/// <reference path=\"./src/core/p5.Element.d.ts\" />\n/// <reference path=\"./src/core/p5.Graphics.d.ts\" />\n/// <reference path=\"./src/image/p5.Image.d.ts\" />\n/// <reference path=\"./src/io/p5.Table.d.ts\" />\n/// <reference path=\"./src/io/p5.TableRow.d.ts\" />\n/// <reference path=\"./src/io/p5.XML.d.ts\" />\n/// <reference path=\"./src/math/p5.Vector.d.ts\" />\n/// <reference path=\"./src/typography/p5.Font.d.ts\" />\n/// <reference path=\"./src/webgl/p5.Framebuffer.d.ts\" />\n/// <reference path=\"./src/webgl/p5.Geometry.d.ts\" />\n/// <reference path=\"./src/webgl/p5.Shader.d.ts\" />\n/// <reference path=\"./src/core/p5.Renderer.d.ts\" />\n/// <reference path=\"./literals.d.ts\" />\n/// <reference path=\"./constants.d.ts\" />\n= p5;\n\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\ninterface p5 extends p5.p5InstanceExtensions {}\n\ndeclare namespace p5 {\n type UNKNOWN_P5_CONSTANT = any;\n // eslint-disable-next-line @typescript-eslint/no-empty-interface\n interface p5InstanceExtensions {}\n}\n\n";
package/dist/artist-dts.js CHANGED
@@ -1 +1 @@
1
- export const artistDts = "// Viji Artist API - Global Type Definitions\n// All types are placed inside declare global {} because this file uses export {}\n// for top-level await support, which makes it a module (where top-level declarations\n// would otherwise be module-scoped, not global).\n\ndeclare global {\n interface AudioAPI {\r\n isConnected: boolean;\r\n volume: {\r\n current: number;\r\n peak: number;\r\n smoothed: number;\r\n };\r\n bands: {\r\n low: number;\r\n lowMid: number;\r\n mid: number;\r\n highMid: number;\r\n high: number;\r\n lowSmoothed: number;\r\n lowMidSmoothed: number;\r\n midSmoothed: number;\r\n highMidSmoothed: number;\r\n highSmoothed: number;\r\n };\r\n beat: {\r\n kick: number;\r\n snare: number;\r\n hat: number;\r\n any: number;\r\n kickSmoothed: number;\r\n snareSmoothed: number;\r\n hatSmoothed: number;\r\n anySmoothed: number;\r\n triggers: {\r\n any: boolean;\r\n kick: boolean;\r\n snare: boolean;\r\n hat: boolean;\r\n };\r\n events: Array<{\r\n type: 'kick' | 'snare' | 'hat';\r\n time: number;\r\n strength: number;\r\n }>;\r\n bpm: number;\r\n confidence: number;\r\n isLocked: boolean;\r\n };\r\n spectral: {\r\n brightness: number;\r\n flatness: number;\r\n };\r\n getFrequencyData: () => Uint8Array;\r\n getWaveform: () => Float32Array;\r\n }\r\n\n interface ButtonConfig {\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface ButtonParameter {\r\n value: boolean;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n interface CaptureFrameOptions {\r\n /** Output format: 'blob' for encoded image, 'bitmap' for GPU-friendly ImageBitmap */\r\n format?: 'blob' | 'bitmap';\r\n /** MIME type for blob output (ignored for bitmap), e.g., 'image/png', 'image/jpeg', 'image/webp' */\r\n type?: string;\r\n /**\r\n * Target resolution.\r\n * - number: scale factor relative to current canvas size (e.g., 0.5 = 50%)\r\n * - { width, height }: exact output size; if aspect ratio differs from canvas,\r\n * the source is center-cropped to match the target aspect ratio before scaling\r\n */\r\n resolution?: number | {\r\n width: number;\r\n height: number;\r\n };\r\n }\r\n\n interface ColorConfig {\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface ColorParameter {\r\n value: string;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n type CVFeature = 'faceDetection' | 'faceMesh' | 'handTracking' | 'poseDetection' | 'bodySegmentation' | 'emotionDetection';\r\n\n type CVFrameRateMode = 'full' | 'half' | 'quarter' | 'eighth';\r\n\n interface DeviceMotionData {\r\n /** Acceleration without gravity (m/s²) */\r\n acceleration: {\r\n x: number | null;\r\n y: number | null;\r\n z: number | null;\r\n } | null;\r\n /** Acceleration including gravity (m/s²) */\r\n accelerationIncludingGravity: {\r\n x: number | null;\r\n y: number | null;\r\n z: number | null;\r\n } | null;\r\n /** Rotation rate (degrees/second) */\r\n rotationRate: {\r\n alpha: number | null;\r\n beta: number | null;\r\n gamma: number | null;\r\n } | null;\r\n /** Interval between updates (milliseconds) */\r\n interval: number;\r\n }\r\n\n interface DeviceOrientationData {\r\n /** Rotation around Z-axis (0-360 degrees, compass heading) */\r\n alpha: number | null;\r\n /** Rotation around X-axis (-180 to 180 degrees, front-to-back tilt) */\r\n beta: number | null;\r\n /** Rotation around Y-axis (-90 to 90 degrees, left-to-right tilt) */\r\n gamma: number | null;\r\n /** True if using magnetometer (compass) for absolute orientation */\r\n absolute: boolean;\r\n }\r\n\n interface DeviceSensorState {\r\n motion: DeviceMotionData | null;\r\n orientation: DeviceOrientationData | null;\r\n }\r\n\n interface DeviceState extends DeviceSensorState {\r\n /** Unique device identifier */\r\n id: string;\r\n /** User-friendly device name */\r\n name: string;\r\n /** Device camera video (null if not available) */\r\n video: VideoAPI | null;\r\n }\r\n\n interface FaceBlendshapes {\r\n browDownLeft: number;\r\n browDownRight: number;\r\n browInnerUp: number;\r\n browOuterUpLeft: number;\r\n browOuterUpRight: number;\r\n cheekPuff: number;\r\n cheekSquintLeft: number;\r\n cheekSquintRight: number;\r\n eyeBlinkLeft: number;\r\n eyeBlinkRight: number;\r\n eyeLookDownLeft: number;\r\n eyeLookDownRight: number;\r\n eyeLookInLeft: number;\r\n eyeLookInRight: number;\r\n eyeLookOutLeft: number;\r\n eyeLookOutRight: number;\r\n eyeLookUpLeft: number;\r\n eyeLookUpRight: number;\r\n eyeSquintLeft: number;\r\n eyeSquintRight: number;\r\n eyeWideLeft: number;\r\n eyeWideRight: number;\r\n jawForward: number;\r\n jawLeft: number;\r\n jawOpen: number;\r\n jawRight: number;\r\n mouthClose: number;\r\n mouthDimpleLeft: number;\r\n mouthDimpleRight: number;\r\n mouthFrownLeft: number;\r\n mouthFrownRight: number;\r\n mouthFunnel: number;\r\n mouthLeft: number;\r\n mouthLowerDownLeft: number;\r\n mouthLowerDownRight: number;\r\n mouthPressLeft: number;\r\n mouthPressRight: number;\r\n mouthPucker: number;\r\n mouthRight: number;\r\n mouthRollLower: number;\r\n mouthRollUpper: number;\r\n mouthShrugLower: number;\r\n mouthShrugUpper: number;\r\n mouthSmileLeft: number;\r\n mouthSmileRight: number;\r\n mouthStretchLeft: number;\r\n mouthStretchRight: number;\r\n mouthUpperUpLeft: number;\r\n mouthUpperUpRight: number;\r\n noseSneerLeft: number;\r\n noseSneerRight: number;\r\n tongueOut: number;\r\n }\r\n\n interface FaceData {\r\n id: number;\r\n bounds: {\r\n x: number;\r\n y: number;\r\n width: number;\r\n height: number;\r\n };\r\n center: {\r\n x: number;\r\n y: number;\r\n };\r\n confidence: number;\r\n landmarks: {\r\n x: number;\r\n y: number;\r\n z?: number;\r\n }[];\r\n expressions: {\r\n neutral: number;\r\n happy: number;\r\n sad: number;\r\n angry: number;\r\n surprised: number;\r\n disgusted: number;\r\n fearful: number;\r\n };\r\n headPose: {\r\n pitch: number;\r\n yaw: number;\r\n roll: number;\r\n };\r\n blendshapes: FaceBlendshapes;\r\n }\r\n\n type FrameRateMode = 'full' | 'half';\r\n\n interface FrequencyBand {\r\n name: string;\r\n min: number;\r\n max: number;\r\n }\r\n\n interface HandData {\r\n id: number;\r\n handedness: 'left' | 'right';\r\n confidence: number;\r\n bounds: {\r\n x: number;\r\n y: number;\r\n width: number;\r\n height: number;\r\n };\r\n landmarks: {\r\n x: number;\r\n y: number;\r\n z: number;\r\n }[];\r\n palm: {\r\n x: number;\r\n y: number;\r\n z: number;\r\n };\r\n fingers: {\r\n thumb: {\r\n tip: {\r\n x: number;\r\n y: number;\r\n z: number;\r\n };\r\n extended: boolean;\r\n };\r\n index: {\r\n tip: {\r\n x: number;\r\n y: number;\r\n z: number;\r\n };\r\n extended: boolean;\r\n };\r\n middle: {\r\n tip: {\r\n x: number;\r\n y: number;\r\n z: number;\r\n };\r\n extended: boolean;\r\n };\r\n ring: {\r\n tip: {\r\n x: number;\r\n y: number;\r\n z: number;\r\n };\r\n extended: boolean;\r\n };\r\n pinky: {\r\n tip: {\r\n x: number;\r\n y: number;\r\n z: number;\r\n };\r\n extended: boolean;\r\n };\r\n };\r\n gestures: {\r\n fist: number;\r\n openPalm: number;\r\n peace: number;\r\n thumbsUp: number;\r\n pointing: number;\r\n };\r\n }\r\n\n interface ImageConfig {\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface ImageParameter {\r\n value: ImageBitmap | null;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n interface KeyboardAPI {\r\n isPressed(key: string): boolean;\r\n wasPressed(key: string): boolean;\r\n wasReleased(key: string): boolean;\r\n activeKeys: Set<string>;\r\n pressedThisFrame: Set<string>;\r\n releasedThisFrame: Set<string>;\r\n lastKeyPressed: string;\r\n lastKeyReleased: string;\r\n shift: boolean;\r\n ctrl: boolean;\r\n alt: boolean;\r\n meta: boolean;\r\n textureData: Uint8Array;\r\n }\r\n\n interface MouseAPI {\r\n x: number;\r\n y: number;\r\n isInCanvas: boolean;\r\n isPressed: boolean;\r\n leftButton: boolean;\r\n rightButton: boolean;\r\n middleButton: boolean;\r\n deltaX: number;\r\n deltaY: number;\r\n wheelDelta: number;\r\n wheelX: number;\r\n wheelY: number;\r\n wasPressed: boolean;\r\n wasReleased: boolean;\r\n wasMoved: boolean;\r\n }\r\n\n interface NumberConfig {\r\n min?: number;\r\n max?: number;\r\n step?: number;\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface NumberParameter {\r\n value: number;\r\n min: number;\r\n max: number;\r\n step: number;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n type ParameterCategory = 'audio' | 'video' | 'interaction' | 'general';\r\n\n interface PointerAPI {\r\n x: number;\r\n y: number;\r\n deltaX: number;\r\n deltaY: number;\r\n isDown: boolean;\r\n wasPressed: boolean;\r\n wasReleased: boolean;\r\n isInCanvas: boolean;\r\n type: 'mouse' | 'touch' | 'none';\r\n }\r\n\n interface PoseData {\r\n confidence: number;\r\n landmarks: {\r\n x: number;\r\n y: number;\r\n z: number;\r\n visibility: number;\r\n }[];\r\n face: {\r\n x: number;\r\n y: number;\r\n }[];\r\n torso: {\r\n x: number;\r\n y: number;\r\n }[];\r\n leftArm: {\r\n x: number;\r\n y: number;\r\n }[];\r\n rightArm: {\r\n x: number;\r\n y: number;\r\n }[];\r\n leftLeg: {\r\n x: number;\r\n y: number;\r\n }[];\r\n rightLeg: {\r\n x: number;\r\n y: number;\r\n }[];\r\n }\r\n\n type Resolution = {\r\n width: number;\r\n height: number;\r\n };\r\n\n interface SegmentationData {\r\n mask: Uint8Array;\r\n width: number;\r\n height: number;\r\n }\r\n\n interface SelectConfig {\r\n options: string[] | number[];\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface SelectParameter {\r\n value: string | number;\r\n options: string[] | number[];\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n interface SliderConfig {\r\n min?: number;\r\n max?: number;\r\n step?: number;\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface SliderParameter {\r\n value: number;\r\n min: number;\r\n max: number;\r\n step: number;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n interface TextConfig {\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n maxLength?: number;\r\n }\r\n\n interface TextParameter {\r\n value: string;\r\n maxLength?: number;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n interface ToggleConfig {\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface ToggleParameter {\r\n value: boolean;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n interface TouchAPI {\r\n points: TouchPoint[];\r\n count: number;\r\n started: TouchPoint[];\r\n moved: TouchPoint[];\r\n ended: TouchPoint[];\r\n primary: TouchPoint | null;\r\n }\r\n\n interface TouchPoint {\r\n id: number;\r\n x: number;\r\n y: number;\r\n pressure: number;\r\n radius: number;\r\n radiusX: number;\r\n radiusY: number;\r\n rotationAngle: number;\r\n force: number;\r\n isInCanvas: boolean;\r\n deltaX: number;\r\n deltaY: number;\r\n velocity: {\r\n x: number;\r\n y: number;\r\n };\r\n isNew: boolean;\r\n isActive: boolean;\r\n isEnding: boolean;\r\n }\r\n\n const VERSION = \"0.2.20\";\r\n\n interface VideoAPI {\r\n isConnected: boolean;\r\n currentFrame: OffscreenCanvas | ImageBitmap | null;\r\n frameWidth: number;\r\n frameHeight: number;\r\n frameRate: number;\r\n getFrameData: () => ImageData | null;\r\n faces: FaceData[];\r\n hands: HandData[];\r\n pose: PoseData | null;\r\n segmentation: SegmentationData | null;\r\n cv: {\r\n enableFaceDetection(enabled: boolean): Promise<void>;\r\n enableFaceMesh(enabled: boolean): Promise<void>;\r\n enableEmotionDetection(enabled: boolean): Promise<void>;\r\n enableHandTracking(enabled: boolean): Promise<void>;\r\n enablePoseDetection(enabled: boolean): Promise<void>;\r\n enableBodySegmentation(enabled: boolean): Promise<void>;\r\n getActiveFeatures(): CVFeature[];\r\n isProcessing(): boolean;\r\n };\r\n }\r\n\n interface VijiAPI {\r\n canvas: OffscreenCanvas;\r\n ctx?: OffscreenCanvasRenderingContext2D;\r\n gl?: WebGLRenderingContext | WebGL2RenderingContext;\r\n width: number;\r\n height: number;\r\n time: number;\r\n deltaTime: number;\r\n frameCount: number;\r\n fps: number;\r\n audio: AudioAPI;\r\n video: VideoAPI;\r\n streams: VideoAPI[];\r\n mouse: MouseAPI;\r\n keyboard: KeyboardAPI;\r\n touches: TouchAPI;\r\n pointer: PointerAPI;\r\n device: DeviceSensorState;\r\n devices: DeviceState[];\r\n slider: (defaultValue: number, config: SliderConfig) => SliderParameter;\r\n color: (defaultValue: string, config: ColorConfig) => ColorParameter;\r\n toggle: (defaultValue: boolean, config: ToggleConfig) => ToggleParameter;\r\n select: (defaultValue: string | number, config: SelectConfig) => SelectParameter;\r\n text: (defaultValue: string, config: TextConfig) => TextParameter;\r\n number: (defaultValue: number, config: NumberConfig) => NumberParameter;\r\n image: (defaultValue: null, config: ImageConfig) => ImageParameter;\r\n button: (config: ButtonConfig) => ButtonParameter;\r\n useContext(type: '2d'): OffscreenCanvasRenderingContext2D;\r\n useContext(type: 'webgl'): WebGLRenderingContext;\r\n useContext(type: 'webgl2'): WebGL2RenderingContext;\r\n }\r\n\n // Runtime global - the main viji object\n const viji: VijiAPI;\n \n // Function type aliases (artists define their own render/setup functions)\n type Render = (viji: VijiAPI) => void;\n type Setup = (viji: VijiAPI) => void;\n}\n\n// Module marker (enables top-level await in artist code)\nexport {};\n";
1
+ export const artistDts = "// Viji Artist API - Global Type Definitions\n// All types are placed inside declare global {} because this file uses export {}\n// for top-level await support, which makes it a module (where top-level declarations\n// would otherwise be module-scoped, not global).\n\ndeclare global {\n interface AudioAPI {\r\n isConnected: boolean;\r\n volume: {\r\n current: number;\r\n peak: number;\r\n smoothed: number;\r\n };\r\n bands: {\r\n low: number;\r\n lowMid: number;\r\n mid: number;\r\n highMid: number;\r\n high: number;\r\n lowSmoothed: number;\r\n lowMidSmoothed: number;\r\n midSmoothed: number;\r\n highMidSmoothed: number;\r\n highSmoothed: number;\r\n };\r\n beat: {\r\n kick: number;\r\n snare: number;\r\n hat: number;\r\n any: number;\r\n kickSmoothed: number;\r\n snareSmoothed: number;\r\n hatSmoothed: number;\r\n anySmoothed: number;\r\n triggers: {\r\n any: boolean;\r\n kick: boolean;\r\n snare: boolean;\r\n hat: boolean;\r\n };\r\n events: Array<{\r\n type: 'kick' | 'snare' | 'hat';\r\n time: number;\r\n strength: number;\r\n }>;\r\n bpm: number;\r\n confidence: number;\r\n isLocked: boolean;\r\n };\r\n spectral: {\r\n brightness: number;\r\n flatness: number;\r\n };\r\n getFrequencyData: () => Uint8Array;\r\n getWaveform: () => Float32Array;\r\n }\r\n\n interface ButtonConfig {\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface ButtonParameter {\r\n value: boolean;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n interface CaptureFrameOptions {\r\n /** Output format: 'blob' for encoded image, 'bitmap' for GPU-friendly ImageBitmap */\r\n format?: 'blob' | 'bitmap';\r\n /** MIME type for blob output (ignored for bitmap), e.g., 'image/png', 'image/jpeg', 'image/webp' */\r\n type?: string;\r\n /**\r\n * Target resolution.\r\n * - number: scale factor relative to current canvas size (e.g., 0.5 = 50%)\r\n * - { width, height }: exact output size; if aspect ratio differs from canvas,\r\n * the source is center-cropped to match the target aspect ratio before scaling\r\n */\r\n resolution?: number | {\r\n width: number;\r\n height: number;\r\n };\r\n }\r\n\n interface ColorConfig {\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface ColorParameter {\r\n value: string;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n type CVFeature = 'faceDetection' | 'faceMesh' | 'handTracking' | 'poseDetection' | 'bodySegmentation' | 'emotionDetection';\r\n\n type CVFrameRateMode = 'full' | 'half' | 'quarter' | 'eighth';\r\n\n interface DeviceMotionData {\r\n /** Acceleration without gravity (m/s²) */\r\n acceleration: {\r\n x: number | null;\r\n y: number | null;\r\n z: number | null;\r\n } | null;\r\n /** Acceleration including gravity (m/s²) */\r\n accelerationIncludingGravity: {\r\n x: number | null;\r\n y: number | null;\r\n z: number | null;\r\n } | null;\r\n /** Rotation rate (degrees/second) */\r\n rotationRate: {\r\n alpha: number | null;\r\n beta: number | null;\r\n gamma: number | null;\r\n } | null;\r\n /** Interval between updates (milliseconds) */\r\n interval: number;\r\n }\r\n\n interface DeviceOrientationData {\r\n /** Rotation around Z-axis (0-360 degrees, compass heading) */\r\n alpha: number | null;\r\n /** Rotation around X-axis (-180 to 180 degrees, front-to-back tilt) */\r\n beta: number | null;\r\n /** Rotation around Y-axis (-90 to 90 degrees, left-to-right tilt) */\r\n gamma: number | null;\r\n /** True if using magnetometer (compass) for absolute orientation */\r\n absolute: boolean;\r\n }\r\n\n interface DeviceSensorState {\r\n motion: DeviceMotionData | null;\r\n orientation: DeviceOrientationData | null;\r\n }\r\n\n interface DeviceState extends DeviceSensorState {\r\n /** Unique device identifier */\r\n id: string;\r\n /** User-friendly device name */\r\n name: string;\r\n /** Device camera video (null if not available) */\r\n video: VideoAPI | null;\r\n /** Device audio stream (null if not available) */\r\n audio: AudioStreamAPI | null;\r\n }\r\n\n interface FaceBlendshapes {\r\n browDownLeft: number;\r\n browDownRight: number;\r\n browInnerUp: number;\r\n browOuterUpLeft: number;\r\n browOuterUpRight: number;\r\n cheekPuff: number;\r\n cheekSquintLeft: number;\r\n cheekSquintRight: number;\r\n eyeBlinkLeft: number;\r\n eyeBlinkRight: number;\r\n eyeLookDownLeft: number;\r\n eyeLookDownRight: number;\r\n eyeLookInLeft: number;\r\n eyeLookInRight: number;\r\n eyeLookOutLeft: number;\r\n eyeLookOutRight: number;\r\n eyeLookUpLeft: number;\r\n eyeLookUpRight: number;\r\n eyeSquintLeft: number;\r\n eyeSquintRight: number;\r\n eyeWideLeft: number;\r\n eyeWideRight: number;\r\n jawForward: number;\r\n jawLeft: number;\r\n jawOpen: number;\r\n jawRight: number;\r\n mouthClose: number;\r\n mouthDimpleLeft: number;\r\n mouthDimpleRight: number;\r\n mouthFrownLeft: number;\r\n mouthFrownRight: number;\r\n mouthFunnel: number;\r\n mouthLeft: number;\r\n mouthLowerDownLeft: number;\r\n mouthLowerDownRight: number;\r\n mouthPressLeft: number;\r\n mouthPressRight: number;\r\n mouthPucker: number;\r\n mouthRight: number;\r\n mouthRollLower: number;\r\n mouthRollUpper: number;\r\n mouthShrugLower: number;\r\n mouthShrugUpper: number;\r\n mouthSmileLeft: number;\r\n mouthSmileRight: number;\r\n mouthStretchLeft: number;\r\n mouthStretchRight: number;\r\n mouthUpperUpLeft: number;\r\n mouthUpperUpRight: number;\r\n noseSneerLeft: number;\r\n noseSneerRight: number;\r\n tongueOut: number;\r\n }\r\n\n interface FaceData {\r\n id: number;\r\n bounds: {\r\n x: number;\r\n y: number;\r\n width: number;\r\n height: number;\r\n };\r\n center: {\r\n x: number;\r\n y: number;\r\n };\r\n confidence: number;\r\n landmarks: {\r\n x: number;\r\n y: number;\r\n z?: number;\r\n }[];\r\n expressions: {\r\n neutral: number;\r\n happy: number;\r\n sad: number;\r\n angry: number;\r\n surprised: number;\r\n disgusted: number;\r\n fearful: number;\r\n };\r\n headPose: {\r\n pitch: number;\r\n yaw: number;\r\n roll: number;\r\n };\r\n blendshapes: FaceBlendshapes;\r\n }\r\n\n type FrameRateMode = 'full' | 'half';\r\n\n interface FrequencyBand {\r\n name: string;\r\n min: number;\r\n max: number;\r\n }\r\n\n interface HandData {\r\n id: number;\r\n handedness: 'left' | 'right';\r\n confidence: number;\r\n bounds: {\r\n x: number;\r\n y: number;\r\n width: number;\r\n height: number;\r\n };\r\n landmarks: {\r\n x: number;\r\n y: number;\r\n z: number;\r\n }[];\r\n palm: {\r\n x: number;\r\n y: number;\r\n z: number;\r\n };\r\n gestures: {\r\n fist: number;\r\n openPalm: number;\r\n peace: number;\r\n thumbsUp: number;\r\n thumbsDown: number;\r\n pointing: number;\r\n iLoveYou: number;\r\n };\r\n }\r\n\n interface ImageConfig {\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface ImageParameter {\r\n value: ImageBitmap | null;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n interface KeyboardAPI {\r\n isPressed(key: string): boolean;\r\n wasPressed(key: string): boolean;\r\n wasReleased(key: string): boolean;\r\n activeKeys: Set<string>;\r\n pressedThisFrame: Set<string>;\r\n releasedThisFrame: Set<string>;\r\n lastKeyPressed: string;\r\n lastKeyReleased: string;\r\n shift: boolean;\r\n ctrl: boolean;\r\n alt: boolean;\r\n meta: boolean;\r\n textureData: Uint8Array;\r\n }\r\n\n interface MouseAPI {\r\n x: number;\r\n y: number;\r\n isInCanvas: boolean;\r\n isPressed: boolean;\r\n leftButton: boolean;\r\n rightButton: boolean;\r\n middleButton: boolean;\r\n deltaX: number;\r\n deltaY: number;\r\n wheelDelta: number;\r\n wheelX: number;\r\n wheelY: number;\r\n wasPressed: boolean;\r\n wasReleased: boolean;\r\n wasMoved: boolean;\r\n }\r\n\n interface NumberConfig {\r\n min?: number;\r\n max?: number;\r\n step?: number;\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface NumberParameter {\r\n value: number;\r\n min: number;\r\n max: number;\r\n step: number;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n type ParameterCategory = 'audio' | 'video' | 'interaction' | 'general';\r\n\n interface PointerAPI {\r\n x: number;\r\n y: number;\r\n deltaX: number;\r\n deltaY: number;\r\n isDown: boolean;\r\n wasPressed: boolean;\r\n wasReleased: boolean;\r\n isInCanvas: boolean;\r\n type: 'mouse' | 'touch' | 'none';\r\n }\r\n\n interface PoseData {\r\n confidence: number;\r\n landmarks: {\r\n x: number;\r\n y: number;\r\n z: number;\r\n visibility: number;\r\n }[];\r\n face: {\r\n x: number;\r\n y: number;\r\n }[];\r\n torso: {\r\n x: number;\r\n y: number;\r\n }[];\r\n leftArm: {\r\n x: number;\r\n y: number;\r\n }[];\r\n rightArm: {\r\n x: number;\r\n y: number;\r\n }[];\r\n leftLeg: {\r\n x: number;\r\n y: number;\r\n }[];\r\n rightLeg: {\r\n x: number;\r\n y: number;\r\n }[];\r\n }\r\n\n type Resolution = {\r\n width: number;\r\n height: number;\r\n };\r\n\n interface SegmentationData {\r\n mask: Uint8Array;\r\n width: number;\r\n height: number;\r\n }\r\n\n interface SelectConfig {\r\n options: string[] | number[];\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface SelectParameter {\r\n value: string | number;\r\n options: string[] | number[];\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n interface SliderConfig {\r\n min?: number;\r\n max?: number;\r\n step?: number;\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface SliderParameter {\r\n value: number;\r\n min: number;\r\n max: number;\r\n step: number;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n interface TextConfig {\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n maxLength?: number;\r\n }\r\n\n interface TextParameter {\r\n value: string;\r\n maxLength?: number;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n interface ToggleConfig {\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface ToggleParameter {\r\n value: boolean;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n interface TouchAPI {\r\n points: TouchPoint[];\r\n count: number;\r\n started: TouchPoint[];\r\n moved: TouchPoint[];\r\n ended: TouchPoint[];\r\n primary: TouchPoint | null;\r\n }\r\n\n interface TouchPoint {\r\n id: number;\r\n x: number;\r\n y: number;\r\n pressure: number;\r\n radius: number;\r\n radiusX: number;\r\n radiusY: number;\r\n rotationAngle: number;\r\n force: number;\r\n isInCanvas: boolean;\r\n deltaX: number;\r\n deltaY: number;\r\n velocity: {\r\n x: number;\r\n y: number;\r\n };\r\n isNew: boolean;\r\n isActive: boolean;\r\n isEnding: boolean;\r\n }\r\n\n const VERSION = \"0.2.20\";\r\n\n interface VideoAPI {\r\n isConnected: boolean;\r\n currentFrame: OffscreenCanvas | ImageBitmap | null;\r\n frameWidth: number;\r\n frameHeight: number;\r\n frameRate: number;\r\n getFrameData: () => ImageData | null;\r\n faces: FaceData[];\r\n hands: HandData[];\r\n pose: PoseData | null;\r\n segmentation: SegmentationData | null;\r\n cv: {\r\n enableFaceDetection(enabled: boolean): Promise<void>;\r\n enableFaceMesh(enabled: boolean): Promise<void>;\r\n enableEmotionDetection(enabled: boolean): Promise<void>;\r\n enableHandTracking(enabled: boolean): Promise<void>;\r\n enablePoseDetection(enabled: boolean): Promise<void>;\r\n enableBodySegmentation(enabled: boolean): Promise<void>;\r\n getActiveFeatures(): CVFeature[];\r\n isProcessing(): boolean;\r\n };\r\n }\r\n\n interface VijiAPI {\r\n canvas: OffscreenCanvas;\r\n ctx?: OffscreenCanvasRenderingContext2D;\r\n gl?: WebGLRenderingContext | WebGL2RenderingContext;\r\n width: number;\r\n height: number;\r\n time: number;\r\n deltaTime: number;\r\n frameCount: number;\r\n fps: number;\r\n audio: AudioAPI;\r\n video: VideoAPI;\r\n videoStreams: VideoAPI[];\r\n audioStreams: AudioStreamAPI[];\r\n mouse: MouseAPI;\r\n keyboard: KeyboardAPI;\r\n touches: TouchAPI;\r\n pointer: PointerAPI;\r\n device: DeviceSensorState;\r\n devices: DeviceState[];\r\n slider: (defaultValue: number, config: SliderConfig) => SliderParameter;\r\n color: (defaultValue: string, config: ColorConfig) => ColorParameter;\r\n toggle: (defaultValue: boolean, config: ToggleConfig) => ToggleParameter;\r\n select: (defaultValue: string | number, config: SelectConfig) => SelectParameter;\r\n text: (defaultValue: string, config: TextConfig) => TextParameter;\r\n number: (defaultValue: number, config: NumberConfig) => NumberParameter;\r\n image: (defaultValue: null, config: ImageConfig) => ImageParameter;\r\n button: (config: ButtonConfig) => ButtonParameter;\r\n useContext(type: '2d'): OffscreenCanvasRenderingContext2D;\r\n useContext(type: 'webgl'): WebGLRenderingContext;\r\n useContext(type: 'webgl2'): WebGL2RenderingContext;\r\n }\r\n\n // Runtime global - the main viji object\n const viji: VijiAPI;\n \n // Function type aliases (artists define their own render/setup functions)\n type Render = (viji: VijiAPI) => void;\n type Setup = (viji: VijiAPI) => void;\n}\n\n// Module marker (enables top-level await in artist code)\nexport {};\n";
package/dist/artist-global.d.ts CHANGED
@@ -152,6 +152,8 @@ declare global {
152
152
  name: string;
153
153
  /** Device camera video (null if not available) */
154
154
  video: VideoAPI | null;
155
+ /** Device audio stream (null if not available) */
156
+ audio: AudioStreamAPI | null;
155
157
  }
156
158
 
157
159
  interface FaceBlendshapes {
@@ -272,54 +274,14 @@ declare global {
272
274
  y: number;
273
275
  z: number;
274
276
  };
275
- fingers: {
276
- thumb: {
277
- tip: {
278
- x: number;
279
- y: number;
280
- z: number;
281
- };
282
- extended: boolean;
283
- };
284
- index: {
285
- tip: {
286
- x: number;
287
- y: number;
288
- z: number;
289
- };
290
- extended: boolean;
291
- };
292
- middle: {
293
- tip: {
294
- x: number;
295
- y: number;
296
- z: number;
297
- };
298
- extended: boolean;
299
- };
300
- ring: {
301
- tip: {
302
- x: number;
303
- y: number;
304
- z: number;
305
- };
306
- extended: boolean;
307
- };
308
- pinky: {
309
- tip: {
310
- x: number;
311
- y: number;
312
- z: number;
313
- };
314
- extended: boolean;
315
- };
316
- };
317
277
  gestures: {
318
278
  fist: number;
319
279
  openPalm: number;
320
280
  peace: number;
321
281
  thumbsUp: number;
282
+ thumbsDown: number;
322
283
  pointing: number;
284
+ iLoveYou: number;
323
285
  };
324
286
  }
325
287
 
@@ -590,7 +552,8 @@ declare global {
590
552
  fps: number;
591
553
  audio: AudioAPI;
592
554
  video: VideoAPI;
593
- streams: VideoAPI[];
555
+ videoStreams: VideoAPI[];
556
+ audioStreams: AudioStreamAPI[];
594
557
  mouse: MouseAPI;
595
558
  keyboard: KeyboardAPI;
596
559
  touches: TouchAPI;
package/dist/artist-jsdoc.d.ts CHANGED
@@ -257,7 +257,8 @@
257
257
  * @property {number} fps - fps property
258
258
  * @property {AudioAPI} audio - audio property
259
259
  * @property {VideoAPI} video - video property
260
- * @property {VideoAPI[]} streams - streams property
260
+ * @property {VideoAPI[]} videoStreams - videoStreams property
261
+ * @property {AudioStreamAPI[]} audioStreams - audioStreams property
261
262
  * @property {MouseAPI} mouse - mouse property
262
263
  * @property {KeyboardAPI} keyboard - keyboard property
263
264
  * @property {TouchAPI} touches - touches property
@@ -405,6 +406,7 @@
405
406
  * @property {string} id - Unique device identifier
406
407
  * @property {string} name - User-friendly device name
407
408
  * @property {VideoAPI |null} video - Device camera video (null if not available)
409
+ * @property {AudioStreamAPI |null} audio - Device audio stream (null if not available)
408
410
  */
409
411
 
410
412
  /**
package/dist/assets/cv-tasks.worker.js CHANGED
@@ -29,7 +29,7 @@ try {
29
29
  // MediaPipe model instances
30
30
  let faceDetector = null;
31
31
  let faceLandmarker = null;
32
- let handLandmarker = null;
32
+ let gestureRecognizer = null;
33
33
  let poseLandmarker = null;
34
34
  let imageSegmenter = null;
35
35
 
@@ -375,31 +375,31 @@ async function initializeFaceLandmarks() {
375
375
  }
376
376
 
377
377
  /**
378
- * Load and initialize Hand Tracking
378
+ * Load and initialize Hand Tracking via GestureRecognizer
379
+ * (provides landmarks + handedness + ML-based gesture classification in one call)
379
380
  */
380
381
  async function initializeHandTracking() {
381
- if (handLandmarker) return;
382
+ if (gestureRecognizer) return;
382
383
 
383
- // Ensure vision runtime is initialized first
384
384
  await initializeVision();
385
385
 
386
386
  try {
387
- log('Loading Hand Landmarker...');
387
+ log('Loading Gesture Recognizer...');
388
388
 
389
389
  const options = {
390
390
  baseOptions: {
391
- modelAssetPath: 'https://storage.googleapis.com/mediapipe-models/hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task',
391
+ modelAssetPath: 'https://storage.googleapis.com/mediapipe-models/gesture_recognizer/gesture_recognizer/float16/1/gesture_recognizer.task',
392
392
  delegate: 'GPU'
393
393
  },
394
394
  runningMode: 'VIDEO',
395
395
  numHands: 2
396
396
  };
397
397
 
398
- const HandLandmarker = self.HandLandmarker || self.module.exports.HandLandmarker || self.exports.HandLandmarker;
399
- handLandmarker = await HandLandmarker.createFromOptions(vision, options);
400
- log('✅ Hand Landmarker loaded');
398
+ const GestureRecognizer = self.GestureRecognizer || self.module.exports.GestureRecognizer || self.exports.GestureRecognizer;
399
+ gestureRecognizer = await GestureRecognizer.createFromOptions(vision, options);
400
+ log('✅ Gesture Recognizer loaded');
401
401
  } catch (error) {
402
- log('❌ Failed to load Hand Landmarker:', error);
402
+ log('❌ Failed to load Gesture Recognizer:', error);
403
403
  throw error;
404
404
  }
405
405
  }
@@ -465,6 +465,59 @@ async function initializeBodySegmentation() {
465
465
  }
466
466
  }
467
467
 
468
+ function computeHandBounds(landmarks) {
469
+ let minX = 1, minY = 1, maxX = 0, maxY = 0;
470
+ for (const p of landmarks) {
471
+ if (p.x < minX) minX = p.x;
472
+ if (p.y < minY) minY = p.y;
473
+ if (p.x > maxX) maxX = p.x;
474
+ if (p.y > maxY) maxY = p.y;
475
+ }
476
+ return { x: minX, y: minY, width: maxX - minX, height: maxY - minY };
477
+ }
478
+
479
+ // Map MediaPipe GestureRecognizer category names to our API field names
480
+ const GESTURE_NAME_MAP = {
481
+ 'Closed_Fist': 'fist',
482
+ 'Open_Palm': 'openPalm',
483
+ 'Victory': 'peace',
484
+ 'Thumb_Up': 'thumbsUp',
485
+ 'Pointing_Up': 'pointing',
486
+ 'Thumb_Down': 'thumbsDown',
487
+ 'ILoveYou': 'iLoveYou'
488
+ };
489
+
490
+ function mapGestures(gestureCategories) {
491
+ const result = { fist: 0, openPalm: 0, peace: 0, thumbsUp: 0, pointing: 0, thumbsDown: 0, iLoveYou: 0 };
492
+ if (!gestureCategories) return result;
493
+ for (const cat of gestureCategories) {
494
+ const key = GESTURE_NAME_MAP[cat.categoryName];
495
+ if (key) result[key] = cat.score;
496
+ }
497
+ return result;
498
+ }
499
+
500
+ // BlazePose landmark index groups for named body parts
501
+ const POSE_FACE_INDICES = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10];
502
+ const POSE_TORSO_INDICES = [11, 12, 23, 24];
503
+ const POSE_LEFT_ARM_INDICES = [11, 13, 15];
504
+ const POSE_RIGHT_ARM_INDICES = [12, 14, 16];
505
+ const POSE_LEFT_LEG_INDICES = [23, 25, 27, 29, 31];
506
+ const POSE_RIGHT_LEG_INDICES = [24, 26, 28, 30, 32];
507
+
508
+ function extractPoseGroup(landmarks, indices) {
509
+ return indices.map(i => {
510
+ const lm = landmarks[i];
511
+ return lm ? { x: lm.x, y: lm.y } : { x: 0, y: 0 };
512
+ });
513
+ }
514
+
515
+ function computePoseConfidence(landmarks) {
516
+ let sum = 0;
517
+ for (const lm of landmarks) sum += (lm.visibility || 0);
518
+ return landmarks.length > 0 ? sum / landmarks.length : 0;
519
+ }
520
+
468
521
  /**
469
522
  * Process video frame with active CV features
470
523
  * @param {ImageData|ImageBitmap} imageInput - Image input (ImageData or ImageBitmap)
@@ -478,7 +531,8 @@ async function processFrame(imageInput, timestamp, features) {
478
531
  // Process face detection
479
532
  if (features.includes('faceDetection') && faceDetector) {
480
533
  const detectionResult = faceDetector.detectForVideo(imageInput, timestamp);
481
- results.faces = detectionResult.detections.map((detection) => ({
534
+ results.faces = detectionResult.detections.map((detection, index) => ({
535
+ id: index,
482
536
  bounds: {
483
537
  x: detection.boundingBox.originX / imageInput.width,
484
538
  y: detection.boundingBox.originY / imageInput.height,
@@ -505,6 +559,7 @@ async function processFrame(imageInput, timestamp, features) {
505
559
 
506
560
  if (!results.faces) {
507
561
  results.faces = [{
562
+ id: 0,
508
563
  bounds: null,
509
564
  center: null,
510
565
  landmarks: [],
@@ -525,7 +580,6 @@ async function processFrame(imageInput, timestamp, features) {
525
580
  results.faces[0].landmarks = mappedLandmarks;
526
581
  results.faces[0].headPose = computeHeadPoseFromLandmarks(landmarks);
527
582
 
528
- // Populate emotion data when emotionDetection is active
529
583
  if (features.includes('emotionDetection') &&
530
584
  landmarkResult.faceBlendshapes &&
531
585
  landmarkResult.faceBlendshapes.length > 0) {
@@ -534,34 +588,55 @@ async function processFrame(imageInput, timestamp, features) {
534
588
  results.faces[0].expressions = mapBlendshapesToEmotions(bs);
535
589
  }
536
590
  }
591
+ } else if (!results.faces) {
592
+ results.faces = [];
537
593
  }
538
594
  }
539
595
 
540
- // Process hand tracking
541
- if (features.includes('handTracking') && handLandmarker) {
542
- const handResult = handLandmarker.detectForVideo(imageInput, timestamp);
543
- results.hands = handResult.landmarks.map((landmarks, index) => ({
544
- landmarks: landmarks.map((landmark) => ({
596
+ // Process hand tracking (GestureRecognizer provides landmarks + gestures in one call)
597
+ if (features.includes('handTracking') && gestureRecognizer) {
598
+ const handResult = gestureRecognizer.recognizeForVideo(imageInput, timestamp);
599
+ results.hands = handResult.landmarks.map((landmarks, index) => {
600
+ const mapped = landmarks.map((landmark) => ({
545
601
  x: landmark.x,
546
602
  y: landmark.y,
547
603
  z: landmark.z || 0
548
- })),
549
- handedness: handResult.handednesses[index]?.[0]?.categoryName || 'Unknown',
550
- confidence: handResult.handednesses[index]?.[0]?.score || 0
551
- }));
604
+ }));
605
+
606
+ const rawHandedness = handResult.handednesses[index]?.[0]?.categoryName || 'unknown';
607
+
608
+ return {
609
+ id: index,
610
+ handedness: rawHandedness.toLowerCase(),
611
+ confidence: handResult.handednesses[index]?.[0]?.score || 0,
612
+ bounds: computeHandBounds(mapped),
613
+ landmarks: mapped,
614
+ palm: mapped[9],
615
+ gestures: mapGestures(handResult.gestures[index])
616
+ };
617
+ });
552
618
  }
553
619
 
554
620
  // Process pose detection
555
621
  if (features.includes('poseDetection') && poseLandmarker) {
556
622
  const poseResult = poseLandmarker.detectForVideo(imageInput, timestamp);
557
623
  if (poseResult.landmarks.length > 0) {
624
+ const poseLandmarks = poseResult.landmarks[0].map((landmark) => ({
625
+ x: landmark.x,
626
+ y: landmark.y,
627
+ z: landmark.z || 0,
628
+ visibility: landmark.visibility || 1
629
+ }));
630
+
558
631
  results.pose = {
559
- landmarks: poseResult.landmarks[0].map((landmark) => ({
560
- x: landmark.x,
561
- y: landmark.y,
562
- z: landmark.z || 0,
563
- visibility: landmark.visibility || 1
564
- })),
632
+ confidence: computePoseConfidence(poseLandmarks),
633
+ landmarks: poseLandmarks,
634
+ face: extractPoseGroup(poseLandmarks, POSE_FACE_INDICES),
635
+ torso: extractPoseGroup(poseLandmarks, POSE_TORSO_INDICES),
636
+ leftArm: extractPoseGroup(poseLandmarks, POSE_LEFT_ARM_INDICES),
637
+ rightArm: extractPoseGroup(poseLandmarks, POSE_RIGHT_ARM_INDICES),
638
+ leftLeg: extractPoseGroup(poseLandmarks, POSE_LEFT_LEG_INDICES),
639
+ rightLeg: extractPoseGroup(poseLandmarks, POSE_RIGHT_LEG_INDICES),
565
640
  worldLandmarks: poseResult.worldLandmarks?.[0]?.map((landmark) => ({
566
641
  x: landmark.x,
567
642
  y: landmark.y,
@@ -569,6 +644,8 @@ async function processFrame(imageInput, timestamp, features) {
569
644
  visibility: landmark.visibility || 1
570
645
  })) || []
571
646
  };
647
+ } else {
648
+ results.pose = null;
572
649
  }
573
650
  }
574
651
 
@@ -577,18 +654,17 @@ async function processFrame(imageInput, timestamp, features) {
577
654
  const segmentResult = imageSegmenter.segment(imageInput);
578
655
  if (segmentResult.categoryMask) {
579
656
  try {
580
- // Extract data before closing the mask
581
657
  results.segmentation = {
582
658
  mask: segmentResult.categoryMask.getAsUint8Array(),
583
659
  width: segmentResult.categoryMask.width,
584
660
  height: segmentResult.categoryMask.height
585
661
  };
586
-
587
662
  log('Segmentation mask:', results.segmentation.width, 'x', results.segmentation.height);
588
663
  } finally {
589
- // CRITICAL: Close MPMask instance to prevent resource leaks
590
664
  segmentResult.categoryMask.close();
591
665
  }
666
+ } else {
667
+ results.segmentation = null;
592
668
  }
593
669
  }
594
670
 
@@ -697,8 +773,8 @@ async function handleConfigUpdateInternal(features) {
697
773
  }
698
774
  break;
699
775
  case 'handTracking':
700
- cleanupPromises.push(cleanupWasmInstance(handLandmarker, 'HandLandmarker'));
701
- handLandmarker = null;
776
+ cleanupPromises.push(cleanupWasmInstance(gestureRecognizer, 'GestureRecognizer'));
777
+ gestureRecognizer = null;
702
778
  break;
703
779
  case 'poseDetection':
704
780
  cleanupPromises.push(cleanupWasmInstance(poseLandmarker, 'PoseLandmarker'));