@viji-dev/core 0.3.25 → 0.3.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/artist-dts-p5.js +1 -1
- package/dist/artist-dts.js +1 -1
- package/dist/artist-global.d.ts +2 -42
- package/dist/assets/cv-tasks.worker.js +108 -32
- package/dist/assets/{viji.worker-Zg128woJ.js → viji.worker-PAf0oIec.js} +240 -144
- package/dist/assets/viji.worker-PAf0oIec.js.map +1 -0
- package/dist/docs-api.js +2082 -504
- package/dist/{essentia-wasm.web-Dq7KVAx1.js → essentia-wasm.web-1yXYrWJ5.js} +2 -2
- package/dist/{essentia-wasm.web-Dq7KVAx1.js.map → essentia-wasm.web-1yXYrWJ5.js.map} +1 -1
- package/dist/{index-C6fMmKQj.js → index-B8BYfP9z.js} +3 -3
- package/dist/index-B8BYfP9z.js.map +1 -0
- package/dist/index.d.ts +2 -42
- package/dist/index.js +1 -1
- package/dist/shader-uniforms.js +126 -11
- package/package.json +1 -1
- package/dist/assets/viji.worker-Zg128woJ.js.map +0 -1
- package/dist/index-C6fMmKQj.js.map +0 -1
package/dist/artist-dts-p5.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export const artistDtsP5 = "// Viji Artist API - Global Type Definitions\n// All types are placed inside declare global {} because this file uses export {}\n// for top-level await support, which makes it a module (where top-level declarations\n// would otherwise be module-scoped, not global).\n\ndeclare global {\n interface AudioAPI {\r\n isConnected: boolean;\r\n volume: {\r\n current: number;\r\n peak: number;\r\n smoothed: number;\r\n };\r\n bands: {\r\n low: number;\r\n lowMid: number;\r\n mid: number;\r\n highMid: number;\r\n high: number;\r\n lowSmoothed: number;\r\n lowMidSmoothed: number;\r\n midSmoothed: number;\r\n highMidSmoothed: number;\r\n highSmoothed: number;\r\n };\r\n beat: {\r\n kick: number;\r\n snare: number;\r\n hat: number;\r\n any: number;\r\n kickSmoothed: number;\r\n snareSmoothed: number;\r\n hatSmoothed: number;\r\n anySmoothed: number;\r\n triggers: {\r\n any: boolean;\r\n kick: boolean;\r\n snare: boolean;\r\n hat: boolean;\r\n };\r\n events: Array<{\r\n type: 'kick' | 'snare' | 'hat';\r\n time: number;\r\n strength: number;\r\n }>;\r\n bpm: number;\r\n confidence: number;\r\n isLocked: boolean;\r\n };\r\n spectral: {\r\n brightness: number;\r\n flatness: number;\r\n };\r\n getFrequencyData: () => Uint8Array;\r\n getWaveform: () => Float32Array;\r\n }\r\n\n interface ButtonConfig {\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface ButtonParameter {\r\n value: boolean;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n interface CaptureFrameOptions {\r\n /** Output format: 'blob' for encoded image, 'bitmap' for GPU-friendly ImageBitmap */\r\n format?: 'blob' | 'bitmap';\r\n /** MIME type for blob output (ignored for bitmap), e.g., 'image/png', 'image/jpeg', 'image/webp' */\r\n type?: string;\r\n /**\r\n * Target resolution.\r\n * - number: scale factor relative to current canvas size (e.g., 0.5 = 50%)\r\n * - { width, height }: exact output size; if aspect ratio differs from canvas,\r\n * the source is center-cropped to match the target aspect ratio before scaling\r\n */\r\n resolution?: number | {\r\n width: number;\r\n height: number;\r\n };\r\n }\r\n\n interface ColorConfig {\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface ColorParameter {\r\n value: string;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n type CVFeature = 'faceDetection' | 'faceMesh' | 'handTracking' | 'poseDetection' | 'bodySegmentation' | 'emotionDetection';\r\n\n type CVFrameRateMode = 'full' | 'half' | 'quarter' | 'eighth';\r\n\n interface DeviceMotionData {\r\n /** Acceleration without gravity (m/s²) */\r\n acceleration: {\r\n x: number | null;\r\n y: number | null;\r\n z: number | null;\r\n } | null;\r\n /** Acceleration including gravity (m/s²) */\r\n accelerationIncludingGravity: {\r\n x: number | null;\r\n y: number | null;\r\n z: number | null;\r\n } | null;\r\n /** Rotation rate (degrees/second) */\r\n rotationRate: {\r\n alpha: number | null;\r\n beta: number | null;\r\n gamma: number | null;\r\n } | null;\r\n /** Interval between updates (milliseconds) */\r\n interval: number;\r\n }\r\n\n interface DeviceOrientationData {\r\n /** Rotation around Z-axis (0-360 degrees, compass heading) */\r\n alpha: number | null;\r\n /** Rotation around X-axis (-180 to 180 degrees, front-to-back tilt) */\r\n beta: number | null;\r\n /** Rotation around Y-axis (-90 to 90 degrees, left-to-right tilt) */\r\n gamma: number | null;\r\n /** True if using magnetometer (compass) for absolute orientation */\r\n absolute: boolean;\r\n }\r\n\n interface DeviceSensorState {\r\n motion: DeviceMotionData | null;\r\n orientation: DeviceOrientationData | null;\r\n }\r\n\n interface DeviceState extends DeviceSensorState {\r\n /** Unique device identifier */\r\n id: string;\r\n /** User-friendly device name */\r\n name: string;\r\n /** Device camera video (null if not available) */\r\n video: VideoAPI | null;\r\n }\r\n\n interface FaceBlendshapes {\r\n browDownLeft: number;\r\n browDownRight: number;\r\n browInnerUp: number;\r\n browOuterUpLeft: number;\r\n browOuterUpRight: number;\r\n cheekPuff: number;\r\n cheekSquintLeft: number;\r\n cheekSquintRight: number;\r\n eyeBlinkLeft: number;\r\n eyeBlinkRight: number;\r\n eyeLookDownLeft: number;\r\n eyeLookDownRight: number;\r\n eyeLookInLeft: number;\r\n eyeLookInRight: number;\r\n eyeLookOutLeft: number;\r\n eyeLookOutRight: number;\r\n eyeLookUpLeft: number;\r\n eyeLookUpRight: number;\r\n eyeSquintLeft: number;\r\n eyeSquintRight: number;\r\n eyeWideLeft: number;\r\n eyeWideRight: number;\r\n jawForward: number;\r\n jawLeft: number;\r\n jawOpen: number;\r\n jawRight: number;\r\n mouthClose: number;\r\n mouthDimpleLeft: number;\r\n mouthDimpleRight: number;\r\n mouthFrownLeft: number;\r\n mouthFrownRight: number;\r\n mouthFunnel: number;\r\n mouthLeft: number;\r\n mouthLowerDownLeft: number;\r\n mouthLowerDownRight: number;\r\n mouthPressLeft: number;\r\n mouthPressRight: number;\r\n mouthPucker: number;\r\n mouthRight: number;\r\n mouthRollLower: number;\r\n mouthRollUpper: number;\r\n mouthShrugLower: number;\r\n mouthShrugUpper: number;\r\n mouthSmileLeft: number;\r\n mouthSmileRight: number;\r\n mouthStretchLeft: number;\r\n mouthStretchRight: number;\r\n mouthUpperUpLeft: number;\r\n mouthUpperUpRight: number;\r\n noseSneerLeft: number;\r\n noseSneerRight: number;\r\n tongueOut: number;\r\n }\r\n\n interface FaceData {\r\n id: number;\r\n bounds: {\r\n x: number;\r\n y: number;\r\n width: number;\r\n height: number;\r\n };\r\n center: {\r\n x: number;\r\n y: number;\r\n };\r\n confidence: number;\r\n landmarks: {\r\n x: number;\r\n y: number;\r\n z?: number;\r\n }[];\r\n expressions: {\r\n neutral: number;\r\n happy: number;\r\n sad: number;\r\n angry: number;\r\n surprised: number;\r\n disgusted: number;\r\n fearful: number;\r\n };\r\n headPose: {\r\n pitch: number;\r\n yaw: number;\r\n roll: number;\r\n };\r\n blendshapes: FaceBlendshapes;\r\n }\r\n\n type FrameRateMode = 'full' | 'half';\r\n\n interface FrequencyBand {\r\n name: string;\r\n min: number;\r\n max: number;\r\n }\r\n\n interface HandData {\r\n id: number;\r\n handedness: 'left' | 'right';\r\n confidence: number;\r\n bounds: {\r\n x: number;\r\n y: number;\r\n width: number;\r\n height: number;\r\n };\r\n landmarks: {\r\n x: number;\r\n y: number;\r\n z: number;\r\n }[];\r\n palm: {\r\n x: number;\r\n y: number;\r\n z: number;\r\n };\r\n fingers: {\r\n thumb: {\r\n tip: {\r\n x: number;\r\n y: number;\r\n z: number;\r\n };\r\n extended: boolean;\r\n };\r\n index: {\r\n tip: {\r\n x: number;\r\n y: number;\r\n z: number;\r\n };\r\n extended: boolean;\r\n };\r\n middle: {\r\n tip: {\r\n x: number;\r\n y: number;\r\n z: number;\r\n };\r\n extended: boolean;\r\n };\r\n ring: {\r\n tip: {\r\n x: number;\r\n y: number;\r\n z: number;\r\n };\r\n extended: boolean;\r\n };\r\n pinky: {\r\n tip: {\r\n x: number;\r\n y: number;\r\n z: number;\r\n };\r\n extended: boolean;\r\n };\r\n };\r\n gestures: {\r\n fist: number;\r\n openPalm: number;\r\n peace: number;\r\n thumbsUp: number;\r\n pointing: number;\r\n };\r\n }\r\n\n interface ImageConfig {\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface ImageParameter {\r\n value: ImageBitmap | null;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n interface KeyboardAPI {\r\n isPressed(key: string): boolean;\r\n wasPressed(key: string): boolean;\r\n wasReleased(key: string): boolean;\r\n activeKeys: Set<string>;\r\n pressedThisFrame: Set<string>;\r\n releasedThisFrame: Set<string>;\r\n lastKeyPressed: string;\r\n lastKeyReleased: string;\r\n shift: boolean;\r\n ctrl: boolean;\r\n alt: boolean;\r\n meta: boolean;\r\n textureData: Uint8Array;\r\n }\r\n\n interface MouseAPI {\r\n x: number;\r\n y: number;\r\n isInCanvas: boolean;\r\n isPressed: boolean;\r\n leftButton: boolean;\r\n rightButton: boolean;\r\n middleButton: boolean;\r\n deltaX: number;\r\n deltaY: number;\r\n wheelDelta: number;\r\n wheelX: number;\r\n wheelY: number;\r\n wasPressed: boolean;\r\n wasReleased: boolean;\r\n wasMoved: boolean;\r\n }\r\n\n interface NumberConfig {\r\n min?: number;\r\n max?: number;\r\n step?: number;\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface NumberParameter {\r\n value: number;\r\n min: number;\r\n max: number;\r\n step: number;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n type ParameterCategory = 'audio' | 'video' | 'interaction' | 'general';\r\n\n interface PointerAPI {\r\n x: number;\r\n y: number;\r\n deltaX: number;\r\n deltaY: number;\r\n isDown: boolean;\r\n wasPressed: boolean;\r\n wasReleased: boolean;\r\n isInCanvas: boolean;\r\n type: 'mouse' | 'touch' | 'none';\r\n }\r\n\n interface PoseData {\r\n confidence: number;\r\n landmarks: {\r\n x: number;\r\n y: number;\r\n z: number;\r\n visibility: number;\r\n /**\n * P5-compatible image object (lazy-loaded, auto-converted from .value)\n * Only available in P5 renderer mode. Use with p5.image() function.\n * Example: p5.image(myImage.p5, x, y)\n */\n p5?: {\n canvas: OffscreenCanvas; // P5.js looks for img.canvas || img.elt\n elt: OffscreenCanvas; // Fallback for compatibility\n width: number; // Logical width\n height: number; // Logical height\n };\n }[];\r\n face: {\r\n x: number;\r\n y: number;\r\n }[];\r\n torso: {\r\n x: number;\r\n y: number;\r\n }[];\r\n leftArm: {\r\n x: number;\r\n y: number;\r\n }[];\r\n rightArm: {\r\n x: number;\r\n y: number;\r\n }[];\r\n leftLeg: {\r\n x: number;\r\n y: number;\r\n }[];\r\n rightLeg: {\r\n x: number;\r\n y: number;\r\n }[];\r\n }\r\n\n type Resolution = {\r\n width: number;\r\n height: number;\r\n };\r\n\n interface SegmentationData {\r\n mask: Uint8Array;\r\n width: number;\r\n height: number;\r\n }\r\n\n interface SelectConfig {\r\n options: string[] | number[];\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface SelectParameter {\r\n value: string | number;\r\n options: string[] | number[];\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n interface SliderConfig {\r\n min?: number;\r\n max?: number;\r\n step?: number;\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface SliderParameter {\r\n value: number;\r\n min: number;\r\n max: number;\r\n step: number;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n interface TextConfig {\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n maxLength?: number;\r\n }\r\n\n interface TextParameter {\r\n value: string;\r\n maxLength?: number;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n interface ToggleConfig {\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface ToggleParameter {\r\n value: boolean;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n interface TouchAPI {\r\n points: TouchPoint[];\r\n count: number;\r\n started: TouchPoint[];\r\n moved: TouchPoint[];\r\n ended: TouchPoint[];\r\n primary: TouchPoint | null;\r\n }\r\n\n interface TouchPoint {\r\n id: number;\r\n x: number;\r\n y: number;\r\n pressure: number;\r\n radius: number;\r\n radiusX: number;\r\n radiusY: number;\r\n rotationAngle: number;\r\n force: number;\r\n isInCanvas: boolean;\r\n deltaX: number;\r\n deltaY: number;\r\n velocity: {\r\n x: number;\r\n y: number;\r\n };\r\n isNew: boolean;\r\n isActive: boolean;\r\n isEnding: boolean;\r\n }\r\n\n const VERSION = \"0.2.20\";\r\n\n interface VideoAPI {\r\n isConnected: boolean;\r\n currentFrame: OffscreenCanvas | ImageBitmap | null;\r\n frameWidth: number;\r\n frameHeight: number;\r\n frameRate: number;\r\n getFrameData: () => ImageData | null;\r\n faces: FaceData[];\r\n hands: HandData[];\r\n pose: PoseData | null;\r\n segmentation: SegmentationData | null;\r\n cv: {\r\n enableFaceDetection(enabled: boolean): Promise<void>;\r\n enableFaceMesh(enabled: boolean): Promise<void>;\r\n enableEmotionDetection(enabled: boolean): Promise<void>;\r\n enableHandTracking(enabled: boolean): Promise<void>;\r\n enablePoseDetection(enabled: boolean): Promise<void>;\r\n enableBodySegmentation(enabled: boolean): Promise<void>;\r\n getActiveFeatures(): CVFeature[];\r\n isProcessing(): boolean;\r\n };\r\n }\r\n\n interface VijiAPI {\r\n canvas: OffscreenCanvas;\r\n ctx?: OffscreenCanvasRenderingContext2D;\r\n gl?: WebGLRenderingContext | WebGL2RenderingContext;\r\n width: number;\r\n height: number;\r\n time: number;\r\n deltaTime: number;\r\n frameCount: number;\r\n fps: number;\r\n audio: AudioAPI;\r\n video: VideoAPI;\r\n streams: VideoAPI[];\r\n mouse: MouseAPI;\r\n keyboard: KeyboardAPI;\r\n touches: TouchAPI;\r\n pointer: PointerAPI;\r\n device: DeviceSensorState;\r\n devices: DeviceState[];\r\n slider: (defaultValue: number, config: SliderConfig) => SliderParameter;\r\n color: (defaultValue: string, config: ColorConfig) => ColorParameter;\r\n toggle: (defaultValue: boolean, config: ToggleConfig) => ToggleParameter;\r\n select: (defaultValue: string | number, config: SelectConfig) => SelectParameter;\r\n text: (defaultValue: string, config: TextConfig) => TextParameter;\r\n number: (defaultValue: number, config: NumberConfig) => NumberParameter;\r\n image: (defaultValue: null, config: ImageConfig) => ImageParameter;\r\n button: (config: ButtonConfig) => ButtonParameter;\r\n useContext(type: '2d'): OffscreenCanvasRenderingContext2D;\r\n useContext(type: 'webgl'): WebGLRenderingContext;\r\n useContext(type: 'webgl2'): WebGL2RenderingContext;\r\n }\r\n\n // Runtime global - the main viji object\n const viji: VijiAPI;\n \n // Function type aliases (artists define their own render/setup functions)\n type Render = (viji: VijiAPI) => void;\n type Setup = (viji: VijiAPI) => void;\n}\n\n// Module marker (enables top-level await in artist code)\nexport {};\n\n\n// ============================================================\n// VIJI + P5.JS HYBRID MODE\n// ============================================================\n// In Viji's P5 renderer, you don't use standard P5 patterns.\n// Instead, you define these Viji-specific functions:\n\n// P5-specific globals (augments the base declare global block)\ndeclare global {\n // P5 instance type for annotations\n type p5 = p5;\n \n // Type aliases for P5 function signatures (artists define their own functions)\n /**\n * P5 render function signature - called every frame\n * @param viji - Viji API (canvas, audio, video, parameters, etc.)\n * @param p5 - P5.js instance with all P5 methods\n */\n type P5Render = (viji: VijiAPI, p5: p5) => void;\n \n /**\n * P5 setup function signature - called once at initialization\n * @param viji - Viji API (canvas, audio, video, parameters, etc.)\n * @param p5 - P5.js instance with all P5 methods\n */\n type P5Setup = (viji: VijiAPI, p5: p5) => void;\n}\n\n// ============================================================\n// IMPORTANT NOTES\n// ============================================================\n// - Don't use: new p5(), preload(), draw(), or P5's setup()\n// - Viji manages the P5 instance lifecycle\n// - Access P5 methods through the 'p5' parameter: p5.rect(), p5.fill(), etc.\n// - Access Viji features through the 'viji' parameter: viji.audio, viji.mouse, etc.\n// - Define parameters at the top level (before setup/render functions)\n\n// ============================================================\n// P5.JS TYPES (namespace and instance methods)\n// ============================================================\n// This file was auto-generated. Please do not edit it.\n/// <reference path=\"./src/accessibility/describe.d.ts\" />\n/// <reference path=\"./src/accessibility/outputs.d.ts\" />\n/// <reference path=\"./src/color/creating_reading.d.ts\" />\n/// <reference path=\"./src/color/setting.d.ts\" />\n/// <reference path=\"./src/core/shape/2d_primitives.d.ts\" />\n/// <reference path=\"./src/core/shape/attributes.d.ts\" />\n/// <reference path=\"./src/core/shape/curves.d.ts\" />\n/// <reference path=\"./src/core/shape/vertex.d.ts\" />\n/// <reference path=\"./src/core/constants.d.ts\" />\n/// <reference path=\"./src/core/environment.d.ts\" />\n/// <reference path=\"./src/core/rendering.d.ts\" />\n/// <reference path=\"./src/core/structure.d.ts\" />\n/// <reference path=\"./src/core/transform.d.ts\" />\n/// <reference path=\"./src/data/local_storage.d.ts\" />\n/// <reference path=\"./src/data/p5.TypedDict.d.ts\" />\n/// <reference path=\"./src/dom/dom.d.ts\" />\n/// <reference path=\"./src/events/acceleration.d.ts\" />\n/// <reference path=\"./src/events/keyboard.d.ts\" />\n/// <reference path=\"./src/events/mouse.d.ts\" />\n/// <reference path=\"./src/events/touch.d.ts\" />\n/// <reference path=\"./src/image/image.d.ts\" />\n/// <reference path=\"./src/image/loading_displaying.d.ts\" />\n/// <reference path=\"./src/image/pixels.d.ts\" />\n/// <reference path=\"./src/io/files.d.ts\" />\n/// <reference path=\"./src/math/calculation.d.ts\" />\n/// <reference path=\"./src/math/math.d.ts\" />\n/// <reference path=\"./src/math/noise.d.ts\" />\n/// <reference path=\"./src/math/random.d.ts\" />\n/// <reference path=\"./src/math/trigonometry.d.ts\" />\n/// <reference path=\"./src/typography/attributes.d.ts\" />\n/// <reference path=\"./src/typography/loading_displaying.d.ts\" />\n/// <reference path=\"./src/utilities/array_functions.d.ts\" />\n/// <reference path=\"./src/utilities/conversion.d.ts\" />\n/// <reference path=\"./src/utilities/string_functions.d.ts\" />\n/// <reference path=\"./src/utilities/time_date.d.ts\" />\n/// <reference path=\"./src/webgl/3d_primitives.d.ts\" />\n/// <reference path=\"./src/webgl/interaction.d.ts\" />\n/// <reference path=\"./src/webgl/light.d.ts\" />\n/// <reference path=\"./src/webgl/loading.d.ts\" />\n/// <reference path=\"./src/webgl/material.d.ts\" />\n/// <reference path=\"./src/webgl/p5.Camera.d.ts\" />\n/// <reference path=\"./src/webgl/p5.RendererGL.Immediate.d.ts\" />\n/// <reference path=\"./src/webgl/p5.RendererGL.d.ts\" />\n/// <reference path=\"./src/color/p5.Color.d.ts\" />\n/// <reference path=\"./src/core/p5.Element.d.ts\" />\n/// <reference path=\"./src/core/p5.Graphics.d.ts\" />\n/// <reference path=\"./src/image/p5.Image.d.ts\" />\n/// <reference path=\"./src/io/p5.Table.d.ts\" />\n/// <reference path=\"./src/io/p5.TableRow.d.ts\" />\n/// <reference path=\"./src/io/p5.XML.d.ts\" />\n/// <reference path=\"./src/math/p5.Vector.d.ts\" />\n/// <reference path=\"./src/typography/p5.Font.d.ts\" />\n/// <reference path=\"./src/webgl/p5.Framebuffer.d.ts\" />\n/// <reference path=\"./src/webgl/p5.Geometry.d.ts\" />\n/// <reference path=\"./src/webgl/p5.Shader.d.ts\" />\n/// <reference path=\"./src/core/p5.Renderer.d.ts\" />\n/// <reference path=\"./literals.d.ts\" />\n/// <reference path=\"./constants.d.ts\" />\n= p5;\n\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\ninterface p5 extends p5.p5InstanceExtensions {}\n\ndeclare namespace p5 {\n type UNKNOWN_P5_CONSTANT = any;\n // eslint-disable-next-line @typescript-eslint/no-empty-interface\n interface p5InstanceExtensions {}\n}\n\n";
|
|
1
|
+
export const artistDtsP5 = "// Viji Artist API - Global Type Definitions\n// All types are placed inside declare global {} because this file uses export {}\n// for top-level await support, which makes it a module (where top-level declarations\n// would otherwise be module-scoped, not global).\n\ndeclare global {\n interface AudioAPI {\r\n isConnected: boolean;\r\n volume: {\r\n current: number;\r\n peak: number;\r\n smoothed: number;\r\n };\r\n bands: {\r\n low: number;\r\n lowMid: number;\r\n mid: number;\r\n highMid: number;\r\n high: number;\r\n lowSmoothed: number;\r\n lowMidSmoothed: number;\r\n midSmoothed: number;\r\n highMidSmoothed: number;\r\n highSmoothed: number;\r\n };\r\n beat: {\r\n kick: number;\r\n snare: number;\r\n hat: number;\r\n any: number;\r\n kickSmoothed: number;\r\n snareSmoothed: number;\r\n hatSmoothed: number;\r\n anySmoothed: number;\r\n triggers: {\r\n any: boolean;\r\n kick: boolean;\r\n snare: boolean;\r\n hat: boolean;\r\n };\r\n events: Array<{\r\n type: 'kick' | 'snare' | 'hat';\r\n time: number;\r\n strength: number;\r\n }>;\r\n bpm: number;\r\n confidence: number;\r\n isLocked: boolean;\r\n };\r\n spectral: {\r\n brightness: number;\r\n flatness: number;\r\n };\r\n getFrequencyData: () => Uint8Array;\r\n getWaveform: () => Float32Array;\r\n }\r\n\n interface ButtonConfig {\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface ButtonParameter {\r\n value: boolean;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n interface CaptureFrameOptions {\r\n /** Output format: 'blob' for encoded image, 'bitmap' for GPU-friendly ImageBitmap */\r\n format?: 'blob' | 'bitmap';\r\n /** MIME type for blob output (ignored for bitmap), e.g., 'image/png', 'image/jpeg', 'image/webp' */\r\n type?: string;\r\n /**\r\n * Target resolution.\r\n * - number: scale factor relative to current canvas size (e.g., 0.5 = 50%)\r\n * - { width, height }: exact output size; if aspect ratio differs from canvas,\r\n * the source is center-cropped to match the target aspect ratio before scaling\r\n */\r\n resolution?: number | {\r\n width: number;\r\n height: number;\r\n };\r\n }\r\n\n interface ColorConfig {\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface ColorParameter {\r\n value: string;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n type CVFeature = 'faceDetection' | 'faceMesh' | 'handTracking' | 'poseDetection' | 'bodySegmentation' | 'emotionDetection';\r\n\n type CVFrameRateMode = 'full' | 'half' | 'quarter' | 'eighth';\r\n\n interface DeviceMotionData {\r\n /** Acceleration without gravity (m/s²) */\r\n acceleration: {\r\n x: number | null;\r\n y: number | null;\r\n z: number | null;\r\n } | null;\r\n /** Acceleration including gravity (m/s²) */\r\n accelerationIncludingGravity: {\r\n x: number | null;\r\n y: number | null;\r\n z: number | null;\r\n } | null;\r\n /** Rotation rate (degrees/second) */\r\n rotationRate: {\r\n alpha: number | null;\r\n beta: number | null;\r\n gamma: number | null;\r\n } | null;\r\n /** Interval between updates (milliseconds) */\r\n interval: number;\r\n }\r\n\n interface DeviceOrientationData {\r\n /** Rotation around Z-axis (0-360 degrees, compass heading) */\r\n alpha: number | null;\r\n /** Rotation around X-axis (-180 to 180 degrees, front-to-back tilt) */\r\n beta: number | null;\r\n /** Rotation around Y-axis (-90 to 90 degrees, left-to-right tilt) */\r\n gamma: number | null;\r\n /** True if using magnetometer (compass) for absolute orientation */\r\n absolute: boolean;\r\n }\r\n\n interface DeviceSensorState {\r\n motion: DeviceMotionData | null;\r\n orientation: DeviceOrientationData | null;\r\n }\r\n\n interface DeviceState extends DeviceSensorState {\r\n /** Unique device identifier */\r\n id: string;\r\n /** User-friendly device name */\r\n name: string;\r\n /** Device camera video (null if not available) */\r\n video: VideoAPI | null;\r\n }\r\n\n interface FaceBlendshapes {\r\n browDownLeft: number;\r\n browDownRight: number;\r\n browInnerUp: number;\r\n browOuterUpLeft: number;\r\n browOuterUpRight: number;\r\n cheekPuff: number;\r\n cheekSquintLeft: number;\r\n cheekSquintRight: number;\r\n eyeBlinkLeft: number;\r\n eyeBlinkRight: number;\r\n eyeLookDownLeft: number;\r\n eyeLookDownRight: number;\r\n eyeLookInLeft: number;\r\n eyeLookInRight: number;\r\n eyeLookOutLeft: number;\r\n eyeLookOutRight: number;\r\n eyeLookUpLeft: number;\r\n eyeLookUpRight: number;\r\n eyeSquintLeft: number;\r\n eyeSquintRight: number;\r\n eyeWideLeft: number;\r\n eyeWideRight: number;\r\n jawForward: number;\r\n jawLeft: number;\r\n jawOpen: number;\r\n jawRight: number;\r\n mouthClose: number;\r\n mouthDimpleLeft: number;\r\n mouthDimpleRight: number;\r\n mouthFrownLeft: number;\r\n mouthFrownRight: number;\r\n mouthFunnel: number;\r\n mouthLeft: number;\r\n mouthLowerDownLeft: number;\r\n mouthLowerDownRight: number;\r\n mouthPressLeft: number;\r\n mouthPressRight: number;\r\n mouthPucker: number;\r\n mouthRight: number;\r\n mouthRollLower: number;\r\n mouthRollUpper: number;\r\n mouthShrugLower: number;\r\n mouthShrugUpper: number;\r\n mouthSmileLeft: number;\r\n mouthSmileRight: number;\r\n mouthStretchLeft: number;\r\n mouthStretchRight: number;\r\n mouthUpperUpLeft: number;\r\n mouthUpperUpRight: number;\r\n noseSneerLeft: number;\r\n noseSneerRight: number;\r\n tongueOut: number;\r\n }\r\n\n interface FaceData {\r\n id: number;\r\n bounds: {\r\n x: number;\r\n y: number;\r\n width: number;\r\n height: number;\r\n };\r\n center: {\r\n x: number;\r\n y: number;\r\n };\r\n confidence: number;\r\n landmarks: {\r\n x: number;\r\n y: number;\r\n z?: number;\r\n }[];\r\n expressions: {\r\n neutral: number;\r\n happy: number;\r\n sad: number;\r\n angry: number;\r\n surprised: number;\r\n disgusted: number;\r\n fearful: number;\r\n };\r\n headPose: {\r\n pitch: number;\r\n yaw: number;\r\n roll: number;\r\n };\r\n blendshapes: FaceBlendshapes;\r\n }\r\n\n type FrameRateMode = 'full' | 'half';\r\n\n interface FrequencyBand {\r\n name: string;\r\n min: number;\r\n max: number;\r\n }\r\n\n interface HandData {\r\n id: number;\r\n handedness: 'left' | 'right';\r\n confidence: number;\r\n bounds: {\r\n x: number;\r\n y: number;\r\n width: number;\r\n height: number;\r\n };\r\n landmarks: {\r\n x: number;\r\n y: number;\r\n z: number;\r\n }[];\r\n palm: {\r\n x: number;\r\n y: number;\r\n z: number;\r\n };\r\n gestures: {\r\n fist: number;\r\n openPalm: number;\r\n peace: number;\r\n thumbsUp: number;\r\n thumbsDown: number;\r\n pointing: number;\r\n iLoveYou: number;\r\n };\r\n }\r\n\n interface ImageConfig {\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface ImageParameter {\r\n value: ImageBitmap | null;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n interface KeyboardAPI {\r\n isPressed(key: string): boolean;\r\n wasPressed(key: string): boolean;\r\n wasReleased(key: string): boolean;\r\n activeKeys: Set<string>;\r\n pressedThisFrame: Set<string>;\r\n releasedThisFrame: Set<string>;\r\n lastKeyPressed: string;\r\n lastKeyReleased: string;\r\n shift: boolean;\r\n ctrl: boolean;\r\n alt: boolean;\r\n meta: boolean;\r\n textureData: Uint8Array;\r\n }\r\n\n interface MouseAPI {\r\n x: number;\r\n y: number;\r\n isInCanvas: boolean;\r\n isPressed: boolean;\r\n leftButton: boolean;\r\n rightButton: boolean;\r\n middleButton: boolean;\r\n deltaX: number;\r\n deltaY: number;\r\n wheelDelta: number;\r\n wheelX: number;\r\n wheelY: number;\r\n wasPressed: boolean;\r\n wasReleased: boolean;\r\n wasMoved: boolean;\r\n }\r\n\n interface NumberConfig {\r\n min?: number;\r\n max?: number;\r\n step?: number;\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface NumberParameter {\r\n value: number;\r\n min: number;\r\n max: number;\r\n step: number;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n type ParameterCategory = 'audio' | 'video' | 'interaction' | 'general';\r\n\n interface PointerAPI {\r\n x: number;\r\n y: number;\r\n deltaX: number;\r\n deltaY: number;\r\n isDown: boolean;\r\n wasPressed: boolean;\r\n wasReleased: boolean;\r\n isInCanvas: boolean;\r\n type: 'mouse' | 'touch' | 'none';\r\n }\r\n\n interface PoseData {\r\n confidence: number;\r\n landmarks: {\r\n x: number;\r\n y: number;\r\n z: number;\r\n visibility: number;\r\n /**\n * P5-compatible image object (lazy-loaded, auto-converted from .value)\n * Only available in P5 renderer mode. Use with p5.image() function.\n * Example: p5.image(myImage.p5, x, y)\n */\n p5?: {\n canvas: OffscreenCanvas; // P5.js looks for img.canvas || img.elt\n elt: OffscreenCanvas; // Fallback for compatibility\n width: number; // Logical width\n height: number; // Logical height\n };\n }[];\r\n face: {\r\n x: number;\r\n y: number;\r\n }[];\r\n torso: {\r\n x: number;\r\n y: number;\r\n }[];\r\n leftArm: {\r\n x: number;\r\n y: number;\r\n }[];\r\n rightArm: {\r\n x: number;\r\n y: number;\r\n }[];\r\n leftLeg: {\r\n x: number;\r\n y: number;\r\n }[];\r\n rightLeg: {\r\n x: number;\r\n y: number;\r\n }[];\r\n }\r\n\n type Resolution = {\r\n width: number;\r\n height: number;\r\n };\r\n\n interface SegmentationData {\r\n mask: Uint8Array;\r\n width: number;\r\n height: number;\r\n }\r\n\n interface SelectConfig {\r\n options: string[] | number[];\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface SelectParameter {\r\n value: string | number;\r\n options: string[] | number[];\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n interface SliderConfig {\r\n min?: number;\r\n max?: number;\r\n step?: number;\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface SliderParameter {\r\n value: number;\r\n min: number;\r\n max: number;\r\n step: number;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n interface TextConfig {\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n maxLength?: number;\r\n }\r\n\n interface TextParameter {\r\n value: string;\r\n maxLength?: number;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n interface ToggleConfig {\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface ToggleParameter {\r\n value: boolean;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n interface TouchAPI {\r\n points: TouchPoint[];\r\n count: number;\r\n started: TouchPoint[];\r\n moved: TouchPoint[];\r\n ended: TouchPoint[];\r\n primary: TouchPoint | null;\r\n }\r\n\n interface TouchPoint {\r\n id: number;\r\n x: number;\r\n y: number;\r\n pressure: number;\r\n radius: number;\r\n radiusX: number;\r\n radiusY: number;\r\n rotationAngle: number;\r\n force: number;\r\n isInCanvas: boolean;\r\n deltaX: number;\r\n deltaY: number;\r\n velocity: {\r\n x: number;\r\n y: number;\r\n };\r\n isNew: boolean;\r\n isActive: boolean;\r\n isEnding: boolean;\r\n }\r\n\n const VERSION = \"0.2.20\";\r\n\n interface VideoAPI {\r\n isConnected: boolean;\r\n currentFrame: OffscreenCanvas | ImageBitmap | null;\r\n frameWidth: number;\r\n frameHeight: number;\r\n frameRate: number;\r\n getFrameData: () => ImageData | null;\r\n faces: FaceData[];\r\n hands: HandData[];\r\n pose: PoseData | null;\r\n segmentation: SegmentationData | null;\r\n cv: {\r\n enableFaceDetection(enabled: boolean): Promise<void>;\r\n enableFaceMesh(enabled: boolean): Promise<void>;\r\n enableEmotionDetection(enabled: boolean): Promise<void>;\r\n enableHandTracking(enabled: boolean): Promise<void>;\r\n enablePoseDetection(enabled: boolean): Promise<void>;\r\n enableBodySegmentation(enabled: boolean): Promise<void>;\r\n getActiveFeatures(): CVFeature[];\r\n isProcessing(): boolean;\r\n };\r\n }\r\n\n interface VijiAPI {\r\n canvas: OffscreenCanvas;\r\n ctx?: OffscreenCanvasRenderingContext2D;\r\n gl?: WebGLRenderingContext | WebGL2RenderingContext;\r\n width: number;\r\n height: number;\r\n time: number;\r\n deltaTime: number;\r\n frameCount: number;\r\n fps: number;\r\n audio: AudioAPI;\r\n video: VideoAPI;\r\n streams: VideoAPI[];\r\n mouse: MouseAPI;\r\n keyboard: KeyboardAPI;\r\n touches: TouchAPI;\r\n pointer: PointerAPI;\r\n device: DeviceSensorState;\r\n devices: DeviceState[];\r\n slider: (defaultValue: number, config: SliderConfig) => SliderParameter;\r\n color: (defaultValue: string, config: ColorConfig) => ColorParameter;\r\n toggle: (defaultValue: boolean, config: ToggleConfig) => ToggleParameter;\r\n select: (defaultValue: string | number, config: SelectConfig) => SelectParameter;\r\n text: (defaultValue: string, config: TextConfig) => TextParameter;\r\n number: (defaultValue: number, config: NumberConfig) => NumberParameter;\r\n image: (defaultValue: null, config: ImageConfig) => ImageParameter;\r\n button: (config: ButtonConfig) => ButtonParameter;\r\n useContext(type: '2d'): OffscreenCanvasRenderingContext2D;\r\n useContext(type: 'webgl'): WebGLRenderingContext;\r\n useContext(type: 'webgl2'): WebGL2RenderingContext;\r\n }\r\n\n // Runtime global - the main viji object\n const viji: VijiAPI;\n \n // Function type aliases (artists define their own render/setup functions)\n type Render = (viji: VijiAPI) => void;\n type Setup = (viji: VijiAPI) => void;\n}\n\n// Module marker (enables top-level await in artist code)\nexport {};\n\n\n// ============================================================\n// VIJI + P5.JS HYBRID MODE\n// ============================================================\n// In Viji's P5 renderer, you don't use standard P5 patterns.\n// Instead, you define these Viji-specific functions:\n\n// P5-specific globals (augments the base declare global block)\ndeclare global {\n // P5 instance type for annotations\n type p5 = p5;\n \n // Type aliases for P5 function signatures (artists define their own functions)\n /**\n * P5 render function signature - called every frame\n * @param viji - Viji API (canvas, audio, video, parameters, etc.)\n * @param p5 - P5.js instance with all P5 methods\n */\n type P5Render = (viji: VijiAPI, p5: p5) => void;\n \n /**\n * P5 setup function signature - called once at initialization\n * @param viji - Viji API (canvas, audio, video, parameters, etc.)\n * @param p5 - P5.js instance with all P5 methods\n */\n type P5Setup = (viji: VijiAPI, p5: p5) => void;\n}\n\n// ============================================================\n// IMPORTANT NOTES\n// ============================================================\n// - Don't use: new p5(), preload(), draw(), or P5's setup()\n// - Viji manages the P5 instance lifecycle\n// - Access P5 methods through the 'p5' parameter: p5.rect(), p5.fill(), etc.\n// - Access Viji features through the 'viji' parameter: viji.audio, viji.mouse, etc.\n// - Define parameters at the top level (before setup/render functions)\n\n// ============================================================\n// P5.JS TYPES (namespace and instance methods)\n// ============================================================\n// This file was auto-generated. Please do not edit it.\n/// <reference path=\"./src/accessibility/describe.d.ts\" />\n/// <reference path=\"./src/accessibility/outputs.d.ts\" />\n/// <reference path=\"./src/color/creating_reading.d.ts\" />\n/// <reference path=\"./src/color/setting.d.ts\" />\n/// <reference path=\"./src/core/shape/2d_primitives.d.ts\" />\n/// <reference path=\"./src/core/shape/attributes.d.ts\" />\n/// <reference path=\"./src/core/shape/curves.d.ts\" />\n/// <reference path=\"./src/core/shape/vertex.d.ts\" />\n/// <reference path=\"./src/core/constants.d.ts\" />\n/// <reference path=\"./src/core/environment.d.ts\" />\n/// <reference path=\"./src/core/rendering.d.ts\" />\n/// <reference path=\"./src/core/structure.d.ts\" />\n/// <reference path=\"./src/core/transform.d.ts\" />\n/// <reference path=\"./src/data/local_storage.d.ts\" />\n/// <reference path=\"./src/data/p5.TypedDict.d.ts\" />\n/// <reference path=\"./src/dom/dom.d.ts\" />\n/// <reference path=\"./src/events/acceleration.d.ts\" />\n/// <reference path=\"./src/events/keyboard.d.ts\" />\n/// <reference path=\"./src/events/mouse.d.ts\" />\n/// <reference path=\"./src/events/touch.d.ts\" />\n/// <reference path=\"./src/image/image.d.ts\" />\n/// <reference path=\"./src/image/loading_displaying.d.ts\" />\n/// <reference path=\"./src/image/pixels.d.ts\" />\n/// <reference path=\"./src/io/files.d.ts\" />\n/// <reference path=\"./src/math/calculation.d.ts\" />\n/// <reference path=\"./src/math/math.d.ts\" />\n/// <reference path=\"./src/math/noise.d.ts\" />\n/// <reference path=\"./src/math/random.d.ts\" />\n/// <reference path=\"./src/math/trigonometry.d.ts\" />\n/// <reference path=\"./src/typography/attributes.d.ts\" />\n/// <reference path=\"./src/typography/loading_displaying.d.ts\" />\n/// <reference path=\"./src/utilities/array_functions.d.ts\" />\n/// <reference path=\"./src/utilities/conversion.d.ts\" />\n/// <reference path=\"./src/utilities/string_functions.d.ts\" />\n/// <reference path=\"./src/utilities/time_date.d.ts\" />\n/// <reference path=\"./src/webgl/3d_primitives.d.ts\" />\n/// <reference path=\"./src/webgl/interaction.d.ts\" />\n/// <reference path=\"./src/webgl/light.d.ts\" />\n/// <reference path=\"./src/webgl/loading.d.ts\" />\n/// <reference path=\"./src/webgl/material.d.ts\" />\n/// <reference path=\"./src/webgl/p5.Camera.d.ts\" />\n/// <reference path=\"./src/webgl/p5.RendererGL.Immediate.d.ts\" />\n/// <reference path=\"./src/webgl/p5.RendererGL.d.ts\" />\n/// <reference path=\"./src/color/p5.Color.d.ts\" />\n/// <reference path=\"./src/core/p5.Element.d.ts\" />\n/// <reference path=\"./src/core/p5.Graphics.d.ts\" />\n/// <reference path=\"./src/image/p5.Image.d.ts\" />\n/// <reference path=\"./src/io/p5.Table.d.ts\" />\n/// <reference path=\"./src/io/p5.TableRow.d.ts\" />\n/// <reference path=\"./src/io/p5.XML.d.ts\" />\n/// <reference path=\"./src/math/p5.Vector.d.ts\" />\n/// <reference path=\"./src/typography/p5.Font.d.ts\" />\n/// <reference path=\"./src/webgl/p5.Framebuffer.d.ts\" />\n/// <reference path=\"./src/webgl/p5.Geometry.d.ts\" />\n/// <reference path=\"./src/webgl/p5.Shader.d.ts\" />\n/// <reference path=\"./src/core/p5.Renderer.d.ts\" />\n/// <reference path=\"./literals.d.ts\" />\n/// <reference path=\"./constants.d.ts\" />\n= p5;\n\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\ninterface p5 extends p5.p5InstanceExtensions {}\n\ndeclare namespace p5 {\n type UNKNOWN_P5_CONSTANT = any;\n // eslint-disable-next-line @typescript-eslint/no-empty-interface\n interface p5InstanceExtensions {}\n}\n\n";
|
package/dist/artist-dts.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export const artistDts = "// Viji Artist API - Global Type Definitions\n// All types are placed inside declare global {} because this file uses export {}\n// for top-level await support, which makes it a module (where top-level declarations\n// would otherwise be module-scoped, not global).\n\ndeclare global {\n interface AudioAPI {\r\n isConnected: boolean;\r\n volume: {\r\n current: number;\r\n peak: number;\r\n smoothed: number;\r\n };\r\n bands: {\r\n low: number;\r\n lowMid: number;\r\n mid: number;\r\n highMid: number;\r\n high: number;\r\n lowSmoothed: number;\r\n lowMidSmoothed: number;\r\n midSmoothed: number;\r\n highMidSmoothed: number;\r\n highSmoothed: number;\r\n };\r\n beat: {\r\n kick: number;\r\n snare: number;\r\n hat: number;\r\n any: number;\r\n kickSmoothed: number;\r\n snareSmoothed: number;\r\n hatSmoothed: number;\r\n anySmoothed: number;\r\n triggers: {\r\n any: boolean;\r\n kick: boolean;\r\n snare: boolean;\r\n hat: boolean;\r\n };\r\n events: Array<{\r\n type: 'kick' | 'snare' | 'hat';\r\n time: number;\r\n strength: number;\r\n }>;\r\n bpm: number;\r\n confidence: number;\r\n isLocked: boolean;\r\n };\r\n spectral: {\r\n brightness: number;\r\n flatness: number;\r\n };\r\n getFrequencyData: () => Uint8Array;\r\n getWaveform: () => Float32Array;\r\n }\r\n\n interface ButtonConfig {\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface ButtonParameter {\r\n value: boolean;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n interface CaptureFrameOptions {\r\n /** Output format: 'blob' for encoded image, 'bitmap' for GPU-friendly ImageBitmap */\r\n format?: 'blob' | 'bitmap';\r\n /** MIME type for blob output (ignored for bitmap), e.g., 'image/png', 'image/jpeg', 'image/webp' */\r\n type?: string;\r\n /**\r\n * Target resolution.\r\n * - number: scale factor relative to current canvas size (e.g., 0.5 = 50%)\r\n * - { width, height }: exact output size; if aspect ratio differs from canvas,\r\n * the source is center-cropped to match the target aspect ratio before scaling\r\n */\r\n resolution?: number | {\r\n width: number;\r\n height: number;\r\n };\r\n }\r\n\n interface ColorConfig {\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface ColorParameter {\r\n value: string;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n type CVFeature = 'faceDetection' | 'faceMesh' | 'handTracking' | 'poseDetection' | 'bodySegmentation' | 'emotionDetection';\r\n\n type CVFrameRateMode = 'full' | 'half' | 'quarter' | 'eighth';\r\n\n interface DeviceMotionData {\r\n /** Acceleration without gravity (m/s²) */\r\n acceleration: {\r\n x: number | null;\r\n y: number | null;\r\n z: number | null;\r\n } | null;\r\n /** Acceleration including gravity (m/s²) */\r\n accelerationIncludingGravity: {\r\n x: number | null;\r\n y: number | null;\r\n z: number | null;\r\n } | null;\r\n /** Rotation rate (degrees/second) */\r\n rotationRate: {\r\n alpha: number | null;\r\n beta: number | null;\r\n gamma: number | null;\r\n } | null;\r\n /** Interval between updates (milliseconds) */\r\n interval: number;\r\n }\r\n\n interface DeviceOrientationData {\r\n /** Rotation around Z-axis (0-360 degrees, compass heading) */\r\n alpha: number | null;\r\n /** Rotation around X-axis (-180 to 180 degrees, front-to-back tilt) */\r\n beta: number | null;\r\n /** Rotation around Y-axis (-90 to 90 degrees, left-to-right tilt) */\r\n gamma: number | null;\r\n /** True if using magnetometer (compass) for absolute orientation */\r\n absolute: boolean;\r\n }\r\n\n interface DeviceSensorState {\r\n motion: DeviceMotionData | null;\r\n orientation: DeviceOrientationData | null;\r\n }\r\n\n interface DeviceState extends DeviceSensorState {\r\n /** Unique device identifier */\r\n id: string;\r\n /** User-friendly device name */\r\n name: string;\r\n /** Device camera video (null if not available) */\r\n video: VideoAPI | null;\r\n }\r\n\n interface FaceBlendshapes {\r\n browDownLeft: number;\r\n browDownRight: number;\r\n browInnerUp: number;\r\n browOuterUpLeft: number;\r\n browOuterUpRight: number;\r\n cheekPuff: number;\r\n cheekSquintLeft: number;\r\n cheekSquintRight: number;\r\n eyeBlinkLeft: number;\r\n eyeBlinkRight: number;\r\n eyeLookDownLeft: number;\r\n eyeLookDownRight: number;\r\n eyeLookInLeft: number;\r\n eyeLookInRight: number;\r\n eyeLookOutLeft: number;\r\n eyeLookOutRight: number;\r\n eyeLookUpLeft: number;\r\n eyeLookUpRight: number;\r\n eyeSquintLeft: number;\r\n eyeSquintRight: number;\r\n eyeWideLeft: number;\r\n eyeWideRight: number;\r\n jawForward: number;\r\n jawLeft: number;\r\n jawOpen: number;\r\n jawRight: number;\r\n mouthClose: number;\r\n mouthDimpleLeft: number;\r\n mouthDimpleRight: number;\r\n mouthFrownLeft: number;\r\n mouthFrownRight: number;\r\n mouthFunnel: number;\r\n mouthLeft: number;\r\n mouthLowerDownLeft: number;\r\n mouthLowerDownRight: number;\r\n mouthPressLeft: number;\r\n mouthPressRight: number;\r\n mouthPucker: number;\r\n mouthRight: number;\r\n mouthRollLower: number;\r\n mouthRollUpper: number;\r\n mouthShrugLower: number;\r\n mouthShrugUpper: number;\r\n mouthSmileLeft: number;\r\n mouthSmileRight: number;\r\n mouthStretchLeft: number;\r\n mouthStretchRight: number;\r\n mouthUpperUpLeft: number;\r\n mouthUpperUpRight: number;\r\n noseSneerLeft: number;\r\n noseSneerRight: number;\r\n tongueOut: number;\r\n }\r\n\n interface FaceData {\r\n id: number;\r\n bounds: {\r\n x: number;\r\n y: number;\r\n width: number;\r\n height: number;\r\n };\r\n center: {\r\n x: number;\r\n y: number;\r\n };\r\n confidence: number;\r\n landmarks: {\r\n x: number;\r\n y: number;\r\n z?: number;\r\n }[];\r\n expressions: {\r\n neutral: number;\r\n happy: number;\r\n sad: number;\r\n angry: number;\r\n surprised: number;\r\n disgusted: number;\r\n fearful: number;\r\n };\r\n headPose: {\r\n pitch: number;\r\n yaw: number;\r\n roll: number;\r\n };\r\n blendshapes: FaceBlendshapes;\r\n }\r\n\n type FrameRateMode = 'full' | 'half';\r\n\n interface FrequencyBand {\r\n name: string;\r\n min: number;\r\n max: number;\r\n }\r\n\n interface HandData {\r\n id: number;\r\n handedness: 'left' | 'right';\r\n confidence: number;\r\n bounds: {\r\n x: number;\r\n y: number;\r\n width: number;\r\n height: number;\r\n };\r\n landmarks: {\r\n x: number;\r\n y: number;\r\n z: number;\r\n }[];\r\n palm: {\r\n x: number;\r\n y: number;\r\n z: number;\r\n };\r\n fingers: {\r\n thumb: {\r\n tip: {\r\n x: number;\r\n y: number;\r\n z: number;\r\n };\r\n extended: boolean;\r\n };\r\n index: {\r\n tip: {\r\n x: number;\r\n y: number;\r\n z: number;\r\n };\r\n extended: boolean;\r\n };\r\n middle: {\r\n tip: {\r\n x: number;\r\n y: number;\r\n z: number;\r\n };\r\n extended: boolean;\r\n };\r\n ring: {\r\n tip: {\r\n x: number;\r\n y: number;\r\n z: number;\r\n };\r\n extended: boolean;\r\n };\r\n pinky: {\r\n tip: {\r\n x: number;\r\n y: number;\r\n z: number;\r\n };\r\n extended: boolean;\r\n };\r\n };\r\n gestures: {\r\n fist: number;\r\n openPalm: number;\r\n peace: number;\r\n thumbsUp: number;\r\n pointing: number;\r\n };\r\n }\r\n\n interface ImageConfig {\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface ImageParameter {\r\n value: ImageBitmap | null;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n interface KeyboardAPI {\r\n isPressed(key: string): boolean;\r\n wasPressed(key: string): boolean;\r\n wasReleased(key: string): boolean;\r\n activeKeys: Set<string>;\r\n pressedThisFrame: Set<string>;\r\n releasedThisFrame: Set<string>;\r\n lastKeyPressed: string;\r\n lastKeyReleased: string;\r\n shift: boolean;\r\n ctrl: boolean;\r\n alt: boolean;\r\n meta: boolean;\r\n textureData: Uint8Array;\r\n }\r\n\n interface MouseAPI {\r\n x: number;\r\n y: number;\r\n isInCanvas: boolean;\r\n isPressed: boolean;\r\n leftButton: boolean;\r\n rightButton: boolean;\r\n middleButton: boolean;\r\n deltaX: number;\r\n deltaY: number;\r\n wheelDelta: number;\r\n wheelX: number;\r\n wheelY: number;\r\n wasPressed: boolean;\r\n wasReleased: boolean;\r\n wasMoved: boolean;\r\n }\r\n\n interface NumberConfig {\r\n min?: number;\r\n max?: number;\r\n step?: number;\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface NumberParameter {\r\n value: number;\r\n min: number;\r\n max: number;\r\n step: number;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n type ParameterCategory = 'audio' | 'video' | 'interaction' | 'general';\r\n\n interface PointerAPI {\r\n x: number;\r\n y: number;\r\n deltaX: number;\r\n deltaY: number;\r\n isDown: boolean;\r\n wasPressed: boolean;\r\n wasReleased: boolean;\r\n isInCanvas: boolean;\r\n type: 'mouse' | 'touch' | 'none';\r\n }\r\n\n interface PoseData {\r\n confidence: number;\r\n landmarks: {\r\n x: number;\r\n y: number;\r\n z: number;\r\n visibility: number;\r\n }[];\r\n face: {\r\n x: number;\r\n y: number;\r\n }[];\r\n torso: {\r\n x: number;\r\n y: number;\r\n }[];\r\n leftArm: {\r\n x: number;\r\n y: number;\r\n }[];\r\n rightArm: {\r\n x: number;\r\n y: number;\r\n }[];\r\n leftLeg: {\r\n x: number;\r\n y: number;\r\n }[];\r\n rightLeg: {\r\n x: number;\r\n y: number;\r\n }[];\r\n }\r\n\n type Resolution = {\r\n width: number;\r\n height: number;\r\n };\r\n\n interface SegmentationData {\r\n mask: Uint8Array;\r\n width: number;\r\n height: number;\r\n }\r\n\n interface SelectConfig {\r\n options: string[] | number[];\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface SelectParameter {\r\n value: string | number;\r\n options: string[] | number[];\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n interface SliderConfig {\r\n min?: number;\r\n max?: number;\r\n step?: number;\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface SliderParameter {\r\n value: number;\r\n min: number;\r\n max: number;\r\n step: number;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n interface TextConfig {\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n maxLength?: number;\r\n }\r\n\n interface TextParameter {\r\n value: string;\r\n maxLength?: number;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n interface ToggleConfig {\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface ToggleParameter {\r\n value: boolean;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n interface TouchAPI {\r\n points: TouchPoint[];\r\n count: number;\r\n started: TouchPoint[];\r\n moved: TouchPoint[];\r\n ended: TouchPoint[];\r\n primary: TouchPoint | null;\r\n }\r\n\n interface TouchPoint {\r\n id: number;\r\n x: number;\r\n y: number;\r\n pressure: number;\r\n radius: number;\r\n radiusX: number;\r\n radiusY: number;\r\n rotationAngle: number;\r\n force: number;\r\n isInCanvas: boolean;\r\n deltaX: number;\r\n deltaY: number;\r\n velocity: {\r\n x: number;\r\n y: number;\r\n };\r\n isNew: boolean;\r\n isActive: boolean;\r\n isEnding: boolean;\r\n }\r\n\n const VERSION = \"0.2.20\";\r\n\n interface VideoAPI {\r\n isConnected: boolean;\r\n currentFrame: OffscreenCanvas | ImageBitmap | null;\r\n frameWidth: number;\r\n frameHeight: number;\r\n frameRate: number;\r\n getFrameData: () => ImageData | null;\r\n faces: FaceData[];\r\n hands: HandData[];\r\n pose: PoseData | null;\r\n segmentation: SegmentationData | null;\r\n cv: {\r\n enableFaceDetection(enabled: boolean): Promise<void>;\r\n enableFaceMesh(enabled: boolean): Promise<void>;\r\n enableEmotionDetection(enabled: boolean): Promise<void>;\r\n enableHandTracking(enabled: boolean): Promise<void>;\r\n enablePoseDetection(enabled: boolean): Promise<void>;\r\n enableBodySegmentation(enabled: boolean): Promise<void>;\r\n getActiveFeatures(): CVFeature[];\r\n isProcessing(): boolean;\r\n };\r\n }\r\n\n interface VijiAPI {\r\n canvas: OffscreenCanvas;\r\n ctx?: OffscreenCanvasRenderingContext2D;\r\n gl?: WebGLRenderingContext | WebGL2RenderingContext;\r\n width: number;\r\n height: number;\r\n time: number;\r\n deltaTime: number;\r\n frameCount: number;\r\n fps: number;\r\n audio: AudioAPI;\r\n video: VideoAPI;\r\n streams: VideoAPI[];\r\n mouse: MouseAPI;\r\n keyboard: KeyboardAPI;\r\n touches: TouchAPI;\r\n pointer: PointerAPI;\r\n device: DeviceSensorState;\r\n devices: DeviceState[];\r\n slider: (defaultValue: number, config: SliderConfig) => SliderParameter;\r\n color: (defaultValue: string, config: ColorConfig) => ColorParameter;\r\n toggle: (defaultValue: boolean, config: ToggleConfig) => ToggleParameter;\r\n select: (defaultValue: string | number, config: SelectConfig) => SelectParameter;\r\n text: (defaultValue: string, config: TextConfig) => TextParameter;\r\n number: (defaultValue: number, config: NumberConfig) => NumberParameter;\r\n image: (defaultValue: null, config: ImageConfig) => ImageParameter;\r\n button: (config: ButtonConfig) => ButtonParameter;\r\n useContext(type: '2d'): OffscreenCanvasRenderingContext2D;\r\n useContext(type: 'webgl'): WebGLRenderingContext;\r\n useContext(type: 'webgl2'): WebGL2RenderingContext;\r\n }\r\n\n // Runtime global - the main viji object\n const viji: VijiAPI;\n \n // Function type aliases (artists define their own render/setup functions)\n type Render = (viji: VijiAPI) => void;\n type Setup = (viji: VijiAPI) => void;\n}\n\n// Module marker (enables top-level await in artist code)\nexport {};\n";
|
|
1
|
+
export const artistDts = "// Viji Artist API - Global Type Definitions\n// All types are placed inside declare global {} because this file uses export {}\n// for top-level await support, which makes it a module (where top-level declarations\n// would otherwise be module-scoped, not global).\n\ndeclare global {\n interface AudioAPI {\r\n isConnected: boolean;\r\n volume: {\r\n current: number;\r\n peak: number;\r\n smoothed: number;\r\n };\r\n bands: {\r\n low: number;\r\n lowMid: number;\r\n mid: number;\r\n highMid: number;\r\n high: number;\r\n lowSmoothed: number;\r\n lowMidSmoothed: number;\r\n midSmoothed: number;\r\n highMidSmoothed: number;\r\n highSmoothed: number;\r\n };\r\n beat: {\r\n kick: number;\r\n snare: number;\r\n hat: number;\r\n any: number;\r\n kickSmoothed: number;\r\n snareSmoothed: number;\r\n hatSmoothed: number;\r\n anySmoothed: number;\r\n triggers: {\r\n any: boolean;\r\n kick: boolean;\r\n snare: boolean;\r\n hat: boolean;\r\n };\r\n events: Array<{\r\n type: 'kick' | 'snare' | 'hat';\r\n time: number;\r\n strength: number;\r\n }>;\r\n bpm: number;\r\n confidence: number;\r\n isLocked: boolean;\r\n };\r\n spectral: {\r\n brightness: number;\r\n flatness: number;\r\n };\r\n getFrequencyData: () => Uint8Array;\r\n getWaveform: () => Float32Array;\r\n }\r\n\n interface ButtonConfig {\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface ButtonParameter {\r\n value: boolean;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n interface CaptureFrameOptions {\r\n /** Output format: 'blob' for encoded image, 'bitmap' for GPU-friendly ImageBitmap */\r\n format?: 'blob' | 'bitmap';\r\n /** MIME type for blob output (ignored for bitmap), e.g., 'image/png', 'image/jpeg', 'image/webp' */\r\n type?: string;\r\n /**\r\n * Target resolution.\r\n * - number: scale factor relative to current canvas size (e.g., 0.5 = 50%)\r\n * - { width, height }: exact output size; if aspect ratio differs from canvas,\r\n * the source is center-cropped to match the target aspect ratio before scaling\r\n */\r\n resolution?: number | {\r\n width: number;\r\n height: number;\r\n };\r\n }\r\n\n interface ColorConfig {\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface ColorParameter {\r\n value: string;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n type CVFeature = 'faceDetection' | 'faceMesh' | 'handTracking' | 'poseDetection' | 'bodySegmentation' | 'emotionDetection';\r\n\n type CVFrameRateMode = 'full' | 'half' | 'quarter' | 'eighth';\r\n\n interface DeviceMotionData {\r\n /** Acceleration without gravity (m/s²) */\r\n acceleration: {\r\n x: number | null;\r\n y: number | null;\r\n z: number | null;\r\n } | null;\r\n /** Acceleration including gravity (m/s²) */\r\n accelerationIncludingGravity: {\r\n x: number | null;\r\n y: number | null;\r\n z: number | null;\r\n } | null;\r\n /** Rotation rate (degrees/second) */\r\n rotationRate: {\r\n alpha: number | null;\r\n beta: number | null;\r\n gamma: number | null;\r\n } | null;\r\n /** Interval between updates (milliseconds) */\r\n interval: number;\r\n }\r\n\n interface DeviceOrientationData {\r\n /** Rotation around Z-axis (0-360 degrees, compass heading) */\r\n alpha: number | null;\r\n /** Rotation around X-axis (-180 to 180 degrees, front-to-back tilt) */\r\n beta: number | null;\r\n /** Rotation around Y-axis (-90 to 90 degrees, left-to-right tilt) */\r\n gamma: number | null;\r\n /** True if using magnetometer (compass) for absolute orientation */\r\n absolute: boolean;\r\n }\r\n\n interface DeviceSensorState {\r\n motion: DeviceMotionData | null;\r\n orientation: DeviceOrientationData | null;\r\n }\r\n\n interface DeviceState extends DeviceSensorState {\r\n /** Unique device identifier */\r\n id: string;\r\n /** User-friendly device name */\r\n name: string;\r\n /** Device camera video (null if not available) */\r\n video: VideoAPI | null;\r\n }\r\n\n interface FaceBlendshapes {\r\n browDownLeft: number;\r\n browDownRight: number;\r\n browInnerUp: number;\r\n browOuterUpLeft: number;\r\n browOuterUpRight: number;\r\n cheekPuff: number;\r\n cheekSquintLeft: number;\r\n cheekSquintRight: number;\r\n eyeBlinkLeft: number;\r\n eyeBlinkRight: number;\r\n eyeLookDownLeft: number;\r\n eyeLookDownRight: number;\r\n eyeLookInLeft: number;\r\n eyeLookInRight: number;\r\n eyeLookOutLeft: number;\r\n eyeLookOutRight: number;\r\n eyeLookUpLeft: number;\r\n eyeLookUpRight: number;\r\n eyeSquintLeft: number;\r\n eyeSquintRight: number;\r\n eyeWideLeft: number;\r\n eyeWideRight: number;\r\n jawForward: number;\r\n jawLeft: number;\r\n jawOpen: number;\r\n jawRight: number;\r\n mouthClose: number;\r\n mouthDimpleLeft: number;\r\n mouthDimpleRight: number;\r\n mouthFrownLeft: number;\r\n mouthFrownRight: number;\r\n mouthFunnel: number;\r\n mouthLeft: number;\r\n mouthLowerDownLeft: number;\r\n mouthLowerDownRight: number;\r\n mouthPressLeft: number;\r\n mouthPressRight: number;\r\n mouthPucker: number;\r\n mouthRight: number;\r\n mouthRollLower: number;\r\n mouthRollUpper: number;\r\n mouthShrugLower: number;\r\n mouthShrugUpper: number;\r\n mouthSmileLeft: number;\r\n mouthSmileRight: number;\r\n mouthStretchLeft: number;\r\n mouthStretchRight: number;\r\n mouthUpperUpLeft: number;\r\n mouthUpperUpRight: number;\r\n noseSneerLeft: number;\r\n noseSneerRight: number;\r\n tongueOut: number;\r\n }\r\n\n interface FaceData {\r\n id: number;\r\n bounds: {\r\n x: number;\r\n y: number;\r\n width: number;\r\n height: number;\r\n };\r\n center: {\r\n x: number;\r\n y: number;\r\n };\r\n confidence: number;\r\n landmarks: {\r\n x: number;\r\n y: number;\r\n z?: number;\r\n }[];\r\n expressions: {\r\n neutral: number;\r\n happy: number;\r\n sad: number;\r\n angry: number;\r\n surprised: number;\r\n disgusted: number;\r\n fearful: number;\r\n };\r\n headPose: {\r\n pitch: number;\r\n yaw: number;\r\n roll: number;\r\n };\r\n blendshapes: FaceBlendshapes;\r\n }\r\n\n type FrameRateMode = 'full' | 'half';\r\n\n interface FrequencyBand {\r\n name: string;\r\n min: number;\r\n max: number;\r\n }\r\n\n interface HandData {\r\n id: number;\r\n handedness: 'left' | 'right';\r\n confidence: number;\r\n bounds: {\r\n x: number;\r\n y: number;\r\n width: number;\r\n height: number;\r\n };\r\n landmarks: {\r\n x: number;\r\n y: number;\r\n z: number;\r\n }[];\r\n palm: {\r\n x: number;\r\n y: number;\r\n z: number;\r\n };\r\n gestures: {\r\n fist: number;\r\n openPalm: number;\r\n peace: number;\r\n thumbsUp: number;\r\n thumbsDown: number;\r\n pointing: number;\r\n iLoveYou: number;\r\n };\r\n }\r\n\n interface ImageConfig {\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface ImageParameter {\r\n value: ImageBitmap | null;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n interface KeyboardAPI {\r\n isPressed(key: string): boolean;\r\n wasPressed(key: string): boolean;\r\n wasReleased(key: string): boolean;\r\n activeKeys: Set<string>;\r\n pressedThisFrame: Set<string>;\r\n releasedThisFrame: Set<string>;\r\n lastKeyPressed: string;\r\n lastKeyReleased: string;\r\n shift: boolean;\r\n ctrl: boolean;\r\n alt: boolean;\r\n meta: boolean;\r\n textureData: Uint8Array;\r\n }\r\n\n interface MouseAPI {\r\n x: number;\r\n y: number;\r\n isInCanvas: boolean;\r\n isPressed: boolean;\r\n leftButton: boolean;\r\n rightButton: boolean;\r\n middleButton: boolean;\r\n deltaX: number;\r\n deltaY: number;\r\n wheelDelta: number;\r\n wheelX: number;\r\n wheelY: number;\r\n wasPressed: boolean;\r\n wasReleased: boolean;\r\n wasMoved: boolean;\r\n }\r\n\n interface NumberConfig {\r\n min?: number;\r\n max?: number;\r\n step?: number;\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface NumberParameter {\r\n value: number;\r\n min: number;\r\n max: number;\r\n step: number;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n type ParameterCategory = 'audio' | 'video' | 'interaction' | 'general';\r\n\n interface PointerAPI {\r\n x: number;\r\n y: number;\r\n deltaX: number;\r\n deltaY: number;\r\n isDown: boolean;\r\n wasPressed: boolean;\r\n wasReleased: boolean;\r\n isInCanvas: boolean;\r\n type: 'mouse' | 'touch' | 'none';\r\n }\r\n\n interface PoseData {\r\n confidence: number;\r\n landmarks: {\r\n x: number;\r\n y: number;\r\n z: number;\r\n visibility: number;\r\n }[];\r\n face: {\r\n x: number;\r\n y: number;\r\n }[];\r\n torso: {\r\n x: number;\r\n y: number;\r\n }[];\r\n leftArm: {\r\n x: number;\r\n y: number;\r\n }[];\r\n rightArm: {\r\n x: number;\r\n y: number;\r\n }[];\r\n leftLeg: {\r\n x: number;\r\n y: number;\r\n }[];\r\n rightLeg: {\r\n x: number;\r\n y: number;\r\n }[];\r\n }\r\n\n type Resolution = {\r\n width: number;\r\n height: number;\r\n };\r\n\n interface SegmentationData {\r\n mask: Uint8Array;\r\n width: number;\r\n height: number;\r\n }\r\n\n interface SelectConfig {\r\n options: string[] | number[];\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface SelectParameter {\r\n value: string | number;\r\n options: string[] | number[];\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n interface SliderConfig {\r\n min?: number;\r\n max?: number;\r\n step?: number;\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface SliderParameter {\r\n value: number;\r\n min: number;\r\n max: number;\r\n step: number;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n interface TextConfig {\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n maxLength?: number;\r\n }\r\n\n interface TextParameter {\r\n value: string;\r\n maxLength?: number;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n interface ToggleConfig {\r\n label: string;\r\n description?: string;\r\n group?: string;\r\n category?: ParameterCategory;\r\n }\r\n\n interface ToggleParameter {\r\n value: boolean;\r\n label: string;\r\n description?: string;\r\n group: string;\r\n category: ParameterCategory;\r\n }\r\n\n interface TouchAPI {\r\n points: TouchPoint[];\r\n count: number;\r\n started: TouchPoint[];\r\n moved: TouchPoint[];\r\n ended: TouchPoint[];\r\n primary: TouchPoint | null;\r\n }\r\n\n interface TouchPoint {\r\n id: number;\r\n x: number;\r\n y: number;\r\n pressure: number;\r\n radius: number;\r\n radiusX: number;\r\n radiusY: number;\r\n rotationAngle: number;\r\n force: number;\r\n isInCanvas: boolean;\r\n deltaX: number;\r\n deltaY: number;\r\n velocity: {\r\n x: number;\r\n y: number;\r\n };\r\n isNew: boolean;\r\n isActive: boolean;\r\n isEnding: boolean;\r\n }\r\n\n const VERSION = \"0.2.20\";\r\n\n interface VideoAPI {\r\n isConnected: boolean;\r\n currentFrame: OffscreenCanvas | ImageBitmap | null;\r\n frameWidth: number;\r\n frameHeight: number;\r\n frameRate: number;\r\n getFrameData: () => ImageData | null;\r\n faces: FaceData[];\r\n hands: HandData[];\r\n pose: PoseData | null;\r\n segmentation: SegmentationData | null;\r\n cv: {\r\n enableFaceDetection(enabled: boolean): Promise<void>;\r\n enableFaceMesh(enabled: boolean): Promise<void>;\r\n enableEmotionDetection(enabled: boolean): Promise<void>;\r\n enableHandTracking(enabled: boolean): Promise<void>;\r\n enablePoseDetection(enabled: boolean): Promise<void>;\r\n enableBodySegmentation(enabled: boolean): Promise<void>;\r\n getActiveFeatures(): CVFeature[];\r\n isProcessing(): boolean;\r\n };\r\n }\r\n\n interface VijiAPI {\r\n canvas: OffscreenCanvas;\r\n ctx?: OffscreenCanvasRenderingContext2D;\r\n gl?: WebGLRenderingContext | WebGL2RenderingContext;\r\n width: number;\r\n height: number;\r\n time: number;\r\n deltaTime: number;\r\n frameCount: number;\r\n fps: number;\r\n audio: AudioAPI;\r\n video: VideoAPI;\r\n streams: VideoAPI[];\r\n mouse: MouseAPI;\r\n keyboard: KeyboardAPI;\r\n touches: TouchAPI;\r\n pointer: PointerAPI;\r\n device: DeviceSensorState;\r\n devices: DeviceState[];\r\n slider: (defaultValue: number, config: SliderConfig) => SliderParameter;\r\n color: (defaultValue: string, config: ColorConfig) => ColorParameter;\r\n toggle: (defaultValue: boolean, config: ToggleConfig) => ToggleParameter;\r\n select: (defaultValue: string | number, config: SelectConfig) => SelectParameter;\r\n text: (defaultValue: string, config: TextConfig) => TextParameter;\r\n number: (defaultValue: number, config: NumberConfig) => NumberParameter;\r\n image: (defaultValue: null, config: ImageConfig) => ImageParameter;\r\n button: (config: ButtonConfig) => ButtonParameter;\r\n useContext(type: '2d'): OffscreenCanvasRenderingContext2D;\r\n useContext(type: 'webgl'): WebGLRenderingContext;\r\n useContext(type: 'webgl2'): WebGL2RenderingContext;\r\n }\r\n\n // Runtime global - the main viji object\n const viji: VijiAPI;\n \n // Function type aliases (artists define their own render/setup functions)\n type Render = (viji: VijiAPI) => void;\n type Setup = (viji: VijiAPI) => void;\n}\n\n// Module marker (enables top-level await in artist code)\nexport {};\n";
|
package/dist/artist-global.d.ts
CHANGED
|
@@ -272,54 +272,14 @@ declare global {
|
|
|
272
272
|
y: number;
|
|
273
273
|
z: number;
|
|
274
274
|
};
|
|
275
|
-
fingers: {
|
|
276
|
-
thumb: {
|
|
277
|
-
tip: {
|
|
278
|
-
x: number;
|
|
279
|
-
y: number;
|
|
280
|
-
z: number;
|
|
281
|
-
};
|
|
282
|
-
extended: boolean;
|
|
283
|
-
};
|
|
284
|
-
index: {
|
|
285
|
-
tip: {
|
|
286
|
-
x: number;
|
|
287
|
-
y: number;
|
|
288
|
-
z: number;
|
|
289
|
-
};
|
|
290
|
-
extended: boolean;
|
|
291
|
-
};
|
|
292
|
-
middle: {
|
|
293
|
-
tip: {
|
|
294
|
-
x: number;
|
|
295
|
-
y: number;
|
|
296
|
-
z: number;
|
|
297
|
-
};
|
|
298
|
-
extended: boolean;
|
|
299
|
-
};
|
|
300
|
-
ring: {
|
|
301
|
-
tip: {
|
|
302
|
-
x: number;
|
|
303
|
-
y: number;
|
|
304
|
-
z: number;
|
|
305
|
-
};
|
|
306
|
-
extended: boolean;
|
|
307
|
-
};
|
|
308
|
-
pinky: {
|
|
309
|
-
tip: {
|
|
310
|
-
x: number;
|
|
311
|
-
y: number;
|
|
312
|
-
z: number;
|
|
313
|
-
};
|
|
314
|
-
extended: boolean;
|
|
315
|
-
};
|
|
316
|
-
};
|
|
317
275
|
gestures: {
|
|
318
276
|
fist: number;
|
|
319
277
|
openPalm: number;
|
|
320
278
|
peace: number;
|
|
321
279
|
thumbsUp: number;
|
|
280
|
+
thumbsDown: number;
|
|
322
281
|
pointing: number;
|
|
282
|
+
iLoveYou: number;
|
|
323
283
|
};
|
|
324
284
|
}
|
|
325
285
|
|
|
@@ -29,7 +29,7 @@ try {
|
|
|
29
29
|
// MediaPipe model instances
|
|
30
30
|
let faceDetector = null;
|
|
31
31
|
let faceLandmarker = null;
|
|
32
|
-
let
|
|
32
|
+
let gestureRecognizer = null;
|
|
33
33
|
let poseLandmarker = null;
|
|
34
34
|
let imageSegmenter = null;
|
|
35
35
|
|
|
@@ -375,31 +375,31 @@ async function initializeFaceLandmarks() {
|
|
|
375
375
|
}
|
|
376
376
|
|
|
377
377
|
/**
|
|
378
|
-
* Load and initialize Hand Tracking
|
|
378
|
+
* Load and initialize Hand Tracking via GestureRecognizer
|
|
379
|
+
* (provides landmarks + handedness + ML-based gesture classification in one call)
|
|
379
380
|
*/
|
|
380
381
|
async function initializeHandTracking() {
|
|
381
|
-
if (
|
|
382
|
+
if (gestureRecognizer) return;
|
|
382
383
|
|
|
383
|
-
// Ensure vision runtime is initialized first
|
|
384
384
|
await initializeVision();
|
|
385
385
|
|
|
386
386
|
try {
|
|
387
|
-
log('Loading
|
|
387
|
+
log('Loading Gesture Recognizer...');
|
|
388
388
|
|
|
389
389
|
const options = {
|
|
390
390
|
baseOptions: {
|
|
391
|
-
modelAssetPath: 'https://storage.googleapis.com/mediapipe-models/
|
|
391
|
+
modelAssetPath: 'https://storage.googleapis.com/mediapipe-models/gesture_recognizer/gesture_recognizer/float16/1/gesture_recognizer.task',
|
|
392
392
|
delegate: 'GPU'
|
|
393
393
|
},
|
|
394
394
|
runningMode: 'VIDEO',
|
|
395
395
|
numHands: 2
|
|
396
396
|
};
|
|
397
397
|
|
|
398
|
-
const
|
|
399
|
-
|
|
400
|
-
log('✅
|
|
398
|
+
const GestureRecognizer = self.GestureRecognizer || self.module.exports.GestureRecognizer || self.exports.GestureRecognizer;
|
|
399
|
+
gestureRecognizer = await GestureRecognizer.createFromOptions(vision, options);
|
|
400
|
+
log('✅ Gesture Recognizer loaded');
|
|
401
401
|
} catch (error) {
|
|
402
|
-
log('❌ Failed to load
|
|
402
|
+
log('❌ Failed to load Gesture Recognizer:', error);
|
|
403
403
|
throw error;
|
|
404
404
|
}
|
|
405
405
|
}
|
|
@@ -465,6 +465,59 @@ async function initializeBodySegmentation() {
|
|
|
465
465
|
}
|
|
466
466
|
}
|
|
467
467
|
|
|
468
|
+
function computeHandBounds(landmarks) {
|
|
469
|
+
let minX = 1, minY = 1, maxX = 0, maxY = 0;
|
|
470
|
+
for (const p of landmarks) {
|
|
471
|
+
if (p.x < minX) minX = p.x;
|
|
472
|
+
if (p.y < minY) minY = p.y;
|
|
473
|
+
if (p.x > maxX) maxX = p.x;
|
|
474
|
+
if (p.y > maxY) maxY = p.y;
|
|
475
|
+
}
|
|
476
|
+
return { x: minX, y: minY, width: maxX - minX, height: maxY - minY };
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
// Map MediaPipe GestureRecognizer category names to our API field names
|
|
480
|
+
const GESTURE_NAME_MAP = {
|
|
481
|
+
'Closed_Fist': 'fist',
|
|
482
|
+
'Open_Palm': 'openPalm',
|
|
483
|
+
'Victory': 'peace',
|
|
484
|
+
'Thumb_Up': 'thumbsUp',
|
|
485
|
+
'Pointing_Up': 'pointing',
|
|
486
|
+
'Thumb_Down': 'thumbsDown',
|
|
487
|
+
'ILoveYou': 'iLoveYou'
|
|
488
|
+
};
|
|
489
|
+
|
|
490
|
+
function mapGestures(gestureCategories) {
|
|
491
|
+
const result = { fist: 0, openPalm: 0, peace: 0, thumbsUp: 0, pointing: 0, thumbsDown: 0, iLoveYou: 0 };
|
|
492
|
+
if (!gestureCategories) return result;
|
|
493
|
+
for (const cat of gestureCategories) {
|
|
494
|
+
const key = GESTURE_NAME_MAP[cat.categoryName];
|
|
495
|
+
if (key) result[key] = cat.score;
|
|
496
|
+
}
|
|
497
|
+
return result;
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
// BlazePose landmark index groups for named body parts
|
|
501
|
+
const POSE_FACE_INDICES = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10];
|
|
502
|
+
const POSE_TORSO_INDICES = [11, 12, 23, 24];
|
|
503
|
+
const POSE_LEFT_ARM_INDICES = [11, 13, 15];
|
|
504
|
+
const POSE_RIGHT_ARM_INDICES = [12, 14, 16];
|
|
505
|
+
const POSE_LEFT_LEG_INDICES = [23, 25, 27, 29, 31];
|
|
506
|
+
const POSE_RIGHT_LEG_INDICES = [24, 26, 28, 30, 32];
|
|
507
|
+
|
|
508
|
+
function extractPoseGroup(landmarks, indices) {
|
|
509
|
+
return indices.map(i => {
|
|
510
|
+
const lm = landmarks[i];
|
|
511
|
+
return lm ? { x: lm.x, y: lm.y } : { x: 0, y: 0 };
|
|
512
|
+
});
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
function computePoseConfidence(landmarks) {
|
|
516
|
+
let sum = 0;
|
|
517
|
+
for (const lm of landmarks) sum += (lm.visibility || 0);
|
|
518
|
+
return landmarks.length > 0 ? sum / landmarks.length : 0;
|
|
519
|
+
}
|
|
520
|
+
|
|
468
521
|
/**
|
|
469
522
|
* Process video frame with active CV features
|
|
470
523
|
* @param {ImageData|ImageBitmap} imageInput - Image input (ImageData or ImageBitmap)
|
|
@@ -478,7 +531,8 @@ async function processFrame(imageInput, timestamp, features) {
|
|
|
478
531
|
// Process face detection
|
|
479
532
|
if (features.includes('faceDetection') && faceDetector) {
|
|
480
533
|
const detectionResult = faceDetector.detectForVideo(imageInput, timestamp);
|
|
481
|
-
results.faces = detectionResult.detections.map((detection) => ({
|
|
534
|
+
results.faces = detectionResult.detections.map((detection, index) => ({
|
|
535
|
+
id: index,
|
|
482
536
|
bounds: {
|
|
483
537
|
x: detection.boundingBox.originX / imageInput.width,
|
|
484
538
|
y: detection.boundingBox.originY / imageInput.height,
|
|
@@ -505,6 +559,7 @@ async function processFrame(imageInput, timestamp, features) {
|
|
|
505
559
|
|
|
506
560
|
if (!results.faces) {
|
|
507
561
|
results.faces = [{
|
|
562
|
+
id: 0,
|
|
508
563
|
bounds: null,
|
|
509
564
|
center: null,
|
|
510
565
|
landmarks: [],
|
|
@@ -525,7 +580,6 @@ async function processFrame(imageInput, timestamp, features) {
|
|
|
525
580
|
results.faces[0].landmarks = mappedLandmarks;
|
|
526
581
|
results.faces[0].headPose = computeHeadPoseFromLandmarks(landmarks);
|
|
527
582
|
|
|
528
|
-
// Populate emotion data when emotionDetection is active
|
|
529
583
|
if (features.includes('emotionDetection') &&
|
|
530
584
|
landmarkResult.faceBlendshapes &&
|
|
531
585
|
landmarkResult.faceBlendshapes.length > 0) {
|
|
@@ -534,34 +588,55 @@ async function processFrame(imageInput, timestamp, features) {
|
|
|
534
588
|
results.faces[0].expressions = mapBlendshapesToEmotions(bs);
|
|
535
589
|
}
|
|
536
590
|
}
|
|
591
|
+
} else if (!results.faces) {
|
|
592
|
+
results.faces = [];
|
|
537
593
|
}
|
|
538
594
|
}
|
|
539
595
|
|
|
540
|
-
// Process hand tracking
|
|
541
|
-
if (features.includes('handTracking') &&
|
|
542
|
-
const handResult =
|
|
543
|
-
results.hands = handResult.landmarks.map((landmarks, index) =>
|
|
544
|
-
|
|
596
|
+
// Process hand tracking (GestureRecognizer provides landmarks + gestures in one call)
|
|
597
|
+
if (features.includes('handTracking') && gestureRecognizer) {
|
|
598
|
+
const handResult = gestureRecognizer.recognizeForVideo(imageInput, timestamp);
|
|
599
|
+
results.hands = handResult.landmarks.map((landmarks, index) => {
|
|
600
|
+
const mapped = landmarks.map((landmark) => ({
|
|
545
601
|
x: landmark.x,
|
|
546
602
|
y: landmark.y,
|
|
547
603
|
z: landmark.z || 0
|
|
548
|
-
}))
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
604
|
+
}));
|
|
605
|
+
|
|
606
|
+
const rawHandedness = handResult.handednesses[index]?.[0]?.categoryName || 'unknown';
|
|
607
|
+
|
|
608
|
+
return {
|
|
609
|
+
id: index,
|
|
610
|
+
handedness: rawHandedness.toLowerCase(),
|
|
611
|
+
confidence: handResult.handednesses[index]?.[0]?.score || 0,
|
|
612
|
+
bounds: computeHandBounds(mapped),
|
|
613
|
+
landmarks: mapped,
|
|
614
|
+
palm: mapped[9],
|
|
615
|
+
gestures: mapGestures(handResult.gestures[index])
|
|
616
|
+
};
|
|
617
|
+
});
|
|
552
618
|
}
|
|
553
619
|
|
|
554
620
|
// Process pose detection
|
|
555
621
|
if (features.includes('poseDetection') && poseLandmarker) {
|
|
556
622
|
const poseResult = poseLandmarker.detectForVideo(imageInput, timestamp);
|
|
557
623
|
if (poseResult.landmarks.length > 0) {
|
|
624
|
+
const poseLandmarks = poseResult.landmarks[0].map((landmark) => ({
|
|
625
|
+
x: landmark.x,
|
|
626
|
+
y: landmark.y,
|
|
627
|
+
z: landmark.z || 0,
|
|
628
|
+
visibility: landmark.visibility || 1
|
|
629
|
+
}));
|
|
630
|
+
|
|
558
631
|
results.pose = {
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
632
|
+
confidence: computePoseConfidence(poseLandmarks),
|
|
633
|
+
landmarks: poseLandmarks,
|
|
634
|
+
face: extractPoseGroup(poseLandmarks, POSE_FACE_INDICES),
|
|
635
|
+
torso: extractPoseGroup(poseLandmarks, POSE_TORSO_INDICES),
|
|
636
|
+
leftArm: extractPoseGroup(poseLandmarks, POSE_LEFT_ARM_INDICES),
|
|
637
|
+
rightArm: extractPoseGroup(poseLandmarks, POSE_RIGHT_ARM_INDICES),
|
|
638
|
+
leftLeg: extractPoseGroup(poseLandmarks, POSE_LEFT_LEG_INDICES),
|
|
639
|
+
rightLeg: extractPoseGroup(poseLandmarks, POSE_RIGHT_LEG_INDICES),
|
|
565
640
|
worldLandmarks: poseResult.worldLandmarks?.[0]?.map((landmark) => ({
|
|
566
641
|
x: landmark.x,
|
|
567
642
|
y: landmark.y,
|
|
@@ -569,6 +644,8 @@ async function processFrame(imageInput, timestamp, features) {
|
|
|
569
644
|
visibility: landmark.visibility || 1
|
|
570
645
|
})) || []
|
|
571
646
|
};
|
|
647
|
+
} else {
|
|
648
|
+
results.pose = null;
|
|
572
649
|
}
|
|
573
650
|
}
|
|
574
651
|
|
|
@@ -577,18 +654,17 @@ async function processFrame(imageInput, timestamp, features) {
|
|
|
577
654
|
const segmentResult = imageSegmenter.segment(imageInput);
|
|
578
655
|
if (segmentResult.categoryMask) {
|
|
579
656
|
try {
|
|
580
|
-
// Extract data before closing the mask
|
|
581
657
|
results.segmentation = {
|
|
582
658
|
mask: segmentResult.categoryMask.getAsUint8Array(),
|
|
583
659
|
width: segmentResult.categoryMask.width,
|
|
584
660
|
height: segmentResult.categoryMask.height
|
|
585
661
|
};
|
|
586
|
-
|
|
587
662
|
log('Segmentation mask:', results.segmentation.width, 'x', results.segmentation.height);
|
|
588
663
|
} finally {
|
|
589
|
-
// CRITICAL: Close MPMask instance to prevent resource leaks
|
|
590
664
|
segmentResult.categoryMask.close();
|
|
591
665
|
}
|
|
666
|
+
} else {
|
|
667
|
+
results.segmentation = null;
|
|
592
668
|
}
|
|
593
669
|
}
|
|
594
670
|
|
|
@@ -697,8 +773,8 @@ async function handleConfigUpdateInternal(features) {
|
|
|
697
773
|
}
|
|
698
774
|
break;
|
|
699
775
|
case 'handTracking':
|
|
700
|
-
cleanupPromises.push(cleanupWasmInstance(
|
|
701
|
-
|
|
776
|
+
cleanupPromises.push(cleanupWasmInstance(gestureRecognizer, 'GestureRecognizer'));
|
|
777
|
+
gestureRecognizer = null;
|
|
702
778
|
break;
|
|
703
779
|
case 'poseDetection':
|
|
704
780
|
cleanupPromises.push(cleanupWasmInstance(poseLandmarker, 'PoseLandmarker'));
|