@clypra/feature-providers 1.0.0

package/.releaserc.json

@@ -0,0 +1,16 @@
+{
+  "branches": ["main"],
+  "plugins": [
+    "@semantic-release/commit-analyzer",
+    "@semantic-release/release-notes-generator",
+    "@semantic-release/npm",
+    [
+      "@semantic-release/git",
+      {
+        "assets": ["package.json"],
+        "message": "chore(release): ${nextRelease.version} [skip ci]\n\n${nextRelease.notes}"
+      }
+    ],
+    "@semantic-release/github"
+  ]
+}

package/package.json

@@ -0,0 +1,42 @@
+{
+  "name": "@clypra/feature-providers",
+  "version": "1.0.0",
+  "description": "Extensible feature providers for Body Effect Lab",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": "./dist/index.js",
+    "./types": "./dist/types.js",
+    "./segmentation": "./dist/segmentation/index.js",
+    "./chroma-key": "./dist/chroma-key/index.js"
+  },
+  "scripts": {
+    "build": "tsup",
+    "dev": "tsup --watch",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "clean": "rm -rf dist",
+    "lint": "tsc --noEmit"
+  },
+  "dependencies": {},
+  "devDependencies": {
+    "@types/node": "^22.14.0",
+    "tsup": "^8.3.5",
+    "typescript": "~5.8.2",
+    "vitest": "^3.2.4"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/AIEraDev/clypra-studio.git",
+    "directory": "packages/feature-providers"
+  },
+  "homepage": "https://github.com/AIEraDev/clypra-studio/tree/main/packages/feature-providers#readme",
+  "bugs": {
+    "url": "https://github.com/AIEraDev/clypra-studio/issues"
+  },
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org/"
+  }
+}

package/src/chroma-key/index.ts

@@ -0,0 +1,176 @@
+/**
+ * @clypra/feature-providers — Chroma Key Provider
+ *
+ * Color-based masking (green screen, blue screen, etc.)
+ */
+
+import type { FeatureProvider, FeatureMap, FeatureMapType, VideoFrame, ProviderConfig } from "../types";
+
+/**
+ * Chroma Key Provider
+ *
+ * Extracts masks based on color keying (e.g., green screen removal)
+ */
+export class ChromaKeyProvider implements FeatureProvider {
+  id = "chroma-key";
+  name = "Chroma Key";
+  outputs: FeatureMapType[] = ["mask" as FeatureMapType];
+
+  config: ProviderConfig = {
+    keyColor: {
+      type: "color",
+      default: "#00FF00",
+      label: "Key Color",
+    },
+    threshold: {
+      type: "number",
+      min: 0,
+      max: 1,
+      default: 0.4,
+      label: "Threshold",
+    },
+    smoothness: {
+      type: "number",
+      min: 0,
+      max: 1,
+      default: 0.2,
+      label: "Edge Smoothness",
+    },
+  };
+
+  private canvas: HTMLCanvasElement | null = null;
+  private ctx: CanvasRenderingContext2D | null = null;
+  private keyColor = { r: 0, g: 255, b: 0 };
+  private threshold = 0.4;
+  private smoothness = 0.2;
+
+  /**
+   * Initialize the provider
+   */
+  async initialize(): Promise<void> {
+    this.canvas = document.createElement("canvas");
+    this.ctx = this.canvas.getContext("2d", {
+      willReadFrequently: true,
+    });
+
+    if (!this.ctx) {
+      throw new Error("Failed to create canvas context");
+    }
+  }
+
+  /**
+   * Process a video frame
+   */
+  async process(frame: VideoFrame): Promise<FeatureMap[]> {
+    if (!this.canvas || !this.ctx) {
+      throw new Error("Provider not initialized");
+    }
+
+    // Get frame dimensions
+    const width = frame instanceof HTMLVideoElement ? frame.videoWidth : frame.width;
+    const height = frame instanceof HTMLVideoElement ? frame.videoHeight : frame.height;
+
+    // Resize canvas if needed
+    if (this.canvas.width !== width || this.canvas.height !== height) {
+      this.canvas.width = width;
+      this.canvas.height = height;
+    }
+
+    // Draw frame to canvas
+    this.ctx.drawImage(frame as any, 0, 0, width, height);
+
+    // Get image data
+    const imageData = this.ctx.getImageData(0, 0, width, height);
+    const data = imageData.data;
+
+    // Apply chroma key
+    for (let i = 0; i < data.length; i += 4) {
+      const r = data[i];
+      const g = data[i + 1];
+      const b = data[i + 2];
+
+      // Calculate color distance
+      const distance = this.colorDistance(r, g, b, this.keyColor.r, this.keyColor.g, this.keyColor.b);
+
+      // Calculate alpha based on distance
+      let alpha = 1.0;
+      if (distance < this.threshold) {
+        // Within threshold - transparent
+        alpha = 0.0;
+      } else if (distance < this.threshold + this.smoothness) {
+        // Edge smoothing
+        alpha = (distance - this.threshold) / this.smoothness;
+      }
+
+      // Set alpha channel (mask)
+      data[i + 3] = Math.floor(alpha * 255);
+    }
+
+    // Put processed data back
+    this.ctx.putImageData(imageData, 0, 0);
+
+    return [
+      {
+        type: "mask" as FeatureMapType,
+        data: {
+          type: "mask",
+          texture: this.canvas,
+          isBinary: false,
+          inverted: false,
+        },
+        metadata: {
+          keyColor: this.keyColor,
+          threshold: this.threshold,
+          smoothness: this.smoothness,
+        },
+      },
+    ];
+  }
+
+  /**
+   * Clean up resources
+   */
+  dispose(): void {
+    this.canvas = null;
+    this.ctx = null;
+  }
+
+  /**
+   * Update configuration
+   */
+  updateConfig(config: Record<string, any>): void {
+    if (config.keyColor) {
+      this.keyColor = this.hexToRgb(config.keyColor);
+    }
+    if (config.threshold !== undefined) {
+      this.threshold = config.threshold;
+    }
+    if (config.smoothness !== undefined) {
+      this.smoothness = config.smoothness;
+    }
+  }
+
+  /**
+   * Calculate color distance (Euclidean)
+   */
+  private colorDistance(r1: number, g1: number, b1: number, r2: number, g2: number, b2: number): number {
+    const dr = (r1 - r2) / 255;
+    const dg = (g1 - g2) / 255;
+    const db = (b1 - b2) / 255;
+    return Math.sqrt(dr * dr + dg * dg + db * db) / Math.sqrt(3);
+  }
+
+  /**
+   * Convert hex color to RGB
+   */
+  private hexToRgb(hex: string): { r: number; g: number; b: number } {
+    const result = /^#?([a-f\d]{2})([a-f\d]{2})([a-f\d]{2})$/i.exec(hex);
+    return result
+      ? {
+          r: parseInt(result[1], 16),
+          g: parseInt(result[2], 16),
+          b: parseInt(result[3], 16),
+        }
+      : { r: 0, g: 255, b: 0 };
+  }
+}

package/src/index.ts

@@ -0,0 +1,36 @@
+/**
+ * @clypra/feature-providers
+ *
+ * Extensible feature providers for Body Effect Lab.
+ *
+ * Feature providers produce feature maps (masks, poses, depth, etc.)
+ * that body effects consume. This architecture makes body effects
+ * future-proof and infinitely extensible.
+ */
+
+// Core types
+export * from "./types";
+
+// Manager
+export { FeatureProviderManager } from "./manager";
+
+// Built-in providers
+export { ChromaKeyProvider } from "./chroma-key";
+export { SegmentationProvider } from "./segmentation";
+
+/**
+ * Create default provider manager with built-in providers
+ */
+import { FeatureProviderManager } from "./manager";
+import { ChromaKeyProvider } from "./chroma-key";
+import { SegmentationProvider } from "./segmentation";
+
+export function createDefaultProviderManager(): FeatureProviderManager {
+  const manager = new FeatureProviderManager();
+
+  // Register built-in providers
+  manager.register(new ChromaKeyProvider());
+  manager.register(new SegmentationProvider());
+
+  return manager;
+}

package/src/manager.ts

@@ -0,0 +1,148 @@
+/**
+ * @clypra/feature-providers — Provider Manager
+ *
+ * Manages feature provider lifecycle and orchestrates feature map generation.
+ */
+
+import type { FeatureProvider, IFeatureProviderManager, FeatureMapType, FeatureMap, VideoFrame } from "./types";
+
+/**
+ * Feature Provider Manager Implementation
+ */
+export class FeatureProviderManager implements IFeatureProviderManager {
+  private providers: Map<string, FeatureProvider> = new Map();
+  private activeProviders: Set<string> = new Set();
+
+  /**
+   * Register a provider
+   */
+  register(provider: FeatureProvider): void {
+    if (this.providers.has(provider.id)) {
+      console.warn(`Provider ${provider.id} is already registered. Replacing.`);
+    }
+    this.providers.set(provider.id, provider);
+  }
+
+  /**
+   * Activate a provider (initialize it)
+   */
+  async activate(providerId: string): Promise<void> {
+    const provider = this.providers.get(providerId);
+    if (!provider) {
+      throw new Error(`Provider ${providerId} not found`);
+    }
+
+    if (this.activeProviders.has(providerId)) {
+      console.warn(`Provider ${providerId} is already active`);
+      return;
+    }
+
+    await provider.initialize();
+    this.activeProviders.add(providerId);
+  }
+
+  /**
+   * Deactivate a provider
+   */
+  deactivate(providerId: string): void {
+    const provider = this.providers.get(providerId);
+    if (!provider) {
+      console.warn(`Provider ${providerId} not found`);
+      return;
+    }
+
+    if (!this.activeProviders.has(providerId)) {
+      console.warn(`Provider ${providerId} is not active`);
+      return;
+    }
+
+    provider.dispose();
+    this.activeProviders.delete(providerId);
+  }
+
+  /**
+   * Process frame with all active providers
+   */
+  async process(frame: VideoFrame): Promise<Map<FeatureMapType, FeatureMap>> {
+    const results = new Map<FeatureMapType, FeatureMap>();
+
+    for (const providerId of this.activeProviders) {
+      const provider = this.providers.get(providerId);
+      if (!provider) continue;
+
+      try {
+        const maps = await provider.process(frame);
+
+        // Store feature maps by type
+        for (const map of maps) {
+          // If multiple providers produce the same type, last one wins
+          // In the future, we could have a priority system or merging strategy
+          results.set(map.type, map);
+        }
+      } catch (error) {
+        console.error(`Error processing with provider ${providerId}:`, error);
+      }
+    }
+
+    return results;
+  }
+
+  /**
+   * Get providers that can produce a specific feature type
+   */
+  getProvidersForFeature(featureType: FeatureMapType): FeatureProvider[] {
+    const providers: FeatureProvider[] = [];
+
+    for (const provider of this.providers.values()) {
+      if (provider.outputs.includes(featureType)) {
+        providers.push(provider);
+      }
+    }
+
+    return providers;
+  }
+
+  /**
+   * Get all registered providers
+   */
+  getAllProviders(): FeatureProvider[] {
+    return Array.from(this.providers.values());
+  }
+
+  /**
+   * Get active providers
+   */
+  getActiveProviders(): FeatureProvider[] {
+    return Array.from(this.activeProviders)
+      .map((id) => this.providers.get(id))
+      .filter((p): p is FeatureProvider => p !== undefined);
+  }
+
+  /**
+   * Get a specific provider by ID
+   */
+  getProvider(providerId: string): FeatureProvider | undefined {
+    return this.providers.get(providerId);
+  }
+
+  /**
+   * Check if a provider is active
+   */
+  isActive(providerId: string): boolean {
+    return this.activeProviders.has(providerId);
+  }
+
+  /**
+   * Dispose all providers
+   */
+  dispose(): void {
+    for (const providerId of this.activeProviders) {
+      const provider = this.providers.get(providerId);
+      if (provider) {
+        provider.dispose();
+      }
+    }
+    this.activeProviders.clear();
+    this.providers.clear();
+  }
+}

package/src/segmentation/index.ts

@@ -0,0 +1,202 @@
+/**
+ * @clypra/feature-providers — Segmentation Provider
+ *
+ * Person segmentation using MediaPipe or similar.
+ * This is a placeholder implementation - actual ML integration will be added later.
+ */
+
+import type { FeatureProvider, FeatureMap, FeatureMapType, VideoFrame, ProviderConfig } from "../types";
+
+/**
+ * Segmentation Provider (Placeholder)
+ *
+ * In production, this would use MediaPipe Selfie Segmentation or similar.
+ * For now, it creates a simple center-weighted mask as a placeholder.
+ */
+export class SegmentationProvider implements FeatureProvider {
+  id = "mediapipe-segmentation";
+  name = "Person Segmentation";
+  outputs: FeatureMapType[] = ["mask" as FeatureMapType];
+
+  config: ProviderConfig = {
+    quality: {
+      type: "select",
+      options: ["low", "medium", "high"],
+      default: "medium",
+      label: "Quality",
+    },
+    edgeSmoothing: {
+      type: "number",
+      min: 0,
+      max: 1,
+      default: 0.5,
+      label: "Edge Smoothing",
+    },
+  };
+
+  private canvas: HTMLCanvasElement | null = null;
+  private ctx: CanvasRenderingContext2D | null = null;
+  private quality: "low" | "medium" | "high" = "medium";
+  private edgeSmoothing = 0.5;
+
+  /**
+   * Initialize the provider
+   *
+   * TODO: Load MediaPipe model here
+   */
+  async initialize(): Promise<void> {
+    this.canvas = document.createElement("canvas");
+    this.ctx = this.canvas.getContext("2d", {
+      willReadFrequently: true,
+    });
+
+    if (!this.ctx) {
+      throw new Error("Failed to create canvas context");
+    }
+
+    // TODO: Initialize MediaPipe
+    // await this.initializeMediaPipe();
+
+    console.log("Segmentation provider initialized (placeholder mode)");
+  }
+
+  /**
+   * Process a video frame
+   *
+   * TODO: Use actual segmentation model
+   */
+  async process(frame: VideoFrame): Promise<FeatureMap[]> {
+    if (!this.canvas || !this.ctx) {
+      throw new Error("Provider not initialized");
+    }
+
+    // Get frame dimensions
+    const width = frame instanceof HTMLVideoElement ? frame.videoWidth : frame.width;
+    const height = frame instanceof HTMLVideoElement ? frame.videoHeight : frame.height;
+
+    // Resize canvas if needed
+    if (this.canvas.width !== width || this.canvas.height !== height) {
+      this.canvas.width = width;
+      this.canvas.height = height;
+    }
+
+    // TODO: Replace with actual segmentation
+    // For now, create a simple radial gradient mask as placeholder
+    this.createPlaceholderMask(width, height);
+
+    return [
+      {
+        type: "mask" as FeatureMapType,
+        data: {
+          type: "mask",
+          texture: this.canvas,
+          isBinary: false,
+          inverted: false,
+        },
+        metadata: {
+          quality: this.quality,
+          edgeSmoothing: this.edgeSmoothing,
+          placeholder: true,
+        },
+      },
+    ];
+  }
+
+  /**
+   * Clean up resources
+   */
+  dispose(): void {
+    // TODO: Dispose MediaPipe resources
+    this.canvas = null;
+    this.ctx = null;
+  }
+
+  /**
+   * Update configuration
+   */
+  updateConfig(config: Record<string, any>): void {
+    if (config.quality) {
+      this.quality = config.quality;
+    }
+    if (config.edgeSmoothing !== undefined) {
+      this.edgeSmoothing = config.edgeSmoothing;
+    }
+  }
+
+  /**
+   * Create a placeholder mask (radial gradient)
+   * This will be replaced with actual segmentation
+   */
+  private createPlaceholderMask(width: number, height: number): void {
+    if (!this.ctx) return;
+
+    const imageData = this.ctx.createImageData(width, height);
+    const data = imageData.data;
+
+    const centerX = width / 2;
+    const centerY = height / 2;
+    const maxRadius = Math.min(width, height) * 0.4;
+
+    for (let y = 0; y < height; y++) {
+      for (let x = 0; x < width; x++) {
+        const i = (y * width + x) * 4;
+
+        // Calculate distance from center
+        const dx = x - centerX;
+        const dy = y - centerY;
+        const distance = Math.sqrt(dx * dx + dy * dy);
+
+        // Calculate alpha based on distance (radial gradient)
+        let alpha = 1.0 - Math.min(distance / maxRadius, 1.0);
+
+        // Apply smoothing
+        if (this.edgeSmoothing > 0) {
+          alpha = this.smoothStep(alpha, this.edgeSmoothing);
+        }
+
+        // Set RGBA (white with varying alpha)
+        data[i] = 255; // R
+        data[i + 1] = 255; // G
+        data[i + 2] = 255; // B
+        data[i + 3] = Math.floor(alpha * 255); // A
+      }
+    }
+
+    this.ctx.putImageData(imageData, 0, 0);
+  }
+
+  /**
+   * Smooth step function for edge smoothing
+   */
+  private smoothStep(value: number, smoothness: number): number {
+    const t = Math.max(0, Math.min(1, (value - (0.5 - smoothness / 2)) / smoothness));
+    return t * t * (3 - 2 * t);
+  }
+}
+
+/**
+ * TODO: Actual MediaPipe integration
+ *
+ * Example of what the real implementation would look like:
+ *
+ * import { SelfieSegmentation } from '@mediapipe/selfie_segmentation';
+ *
+ * async initializeMediaPipe() {
+ *   this.segmenter = new SelfieSegmentation({
+ *     locateFile: (file) => {
+ *       return `https://cdn.jsdelivr.net/npm/@mediapipe/selfie_segmentation/${file}`;
+ *     }
+ *   });
+ *
+ *   this.segmenter.setOptions({
+ *     modelSelection: this.quality === 'high' ? 1 : 0,
+ *   });
+ *
+ *   await this.segmenter.initialize();
+ * }
+ *
+ * async processWithMediaPipe(frame: VideoFrame): Promise<ImageData> {
+ *   const results = await this.segmenter.send({ image: frame });
+ *   return results.segmentationMask;
+ * }
+ */

package/src/types.ts

@@ -0,0 +1,195 @@
+/**
+ * @clypra/feature-providers — Type Definitions
+ *
+ * Core types for extensible feature providers used in Body Effect Lab.
+ */
+
+/**
+ * Types of feature maps that providers can produce
+ */
+export enum FeatureMapType {
+  // Segmentation
+  Mask = "mask",
+  MultiMask = "multi-mask",
+
+  // Spatial
+  Pose = "pose",
+  FaceMesh = "face-mesh",
+  Skeleton = "skeleton",
+  Hands = "hands",
+
+  // Depth
+  Depth = "depth",
+  Normal = "normal",
+
+  // Motion
+  OpticalFlow = "optical-flow",
+  Motion = "motion",
+
+  // Semantic
+  Hair = "hair",
+  Eyes = "eyes",
+  Skin = "skin",
+
+  // Identity
+  PersonID = "person-id",
+}
+
+/**
+ * Base feature map interface
+ */
+export interface FeatureMap {
+  type: FeatureMapType;
+  data: FeatureMapData;
+  metadata?: Record<string, any>;
+}
+
+/**
+ * Union of all feature map data types
+ */
+export type FeatureMapData = MaskData | MultiMaskData | PoseData | DepthData | FlowData | MeshData;
+
+/**
+ * Mask data (binary or alpha mask)
+ */
+export interface MaskData {
+  type: "mask";
+  texture: HTMLCanvasElement | ImageBitmap | HTMLVideoElement;
+  isBinary: boolean;
+  inverted?: boolean;
+}
+
+/**
+ * Multiple masks with IDs (for multi-person scenarios)
+ */
+export interface MultiMaskData {
+  type: "multi-mask";
+  masks: Array<{
+    id: string;
+    texture: HTMLCanvasElement | ImageBitmap;
+    confidence: number;
+  }>;
+}
+
+/**
+ * 2D keypoints (pose, hands, face)
+ */
+export interface PoseData {
+  type: "pose";
+  keypoints: Array<{
+    name: string;
+    x: number; // Normalized 0-1
+    y: number; // Normalized 0-1
+    confidence: number;
+    visible: boolean;
+  }>;
+  skeleton?: Array<[string, string]>; // Connections between keypoints
+}
+
+/**
+ * Depth map data
+ */
+export interface DepthData {
+  type: "depth";
+  texture: HTMLCanvasElement | ImageBitmap;
+  minDepth: number;
+  maxDepth: number;
+  confidence?: HTMLCanvasElement | ImageBitmap;
+}
+
+/**
+ * Optical flow data
+ */
+export interface FlowData {
+  type: "optical-flow";
+  texture: HTMLCanvasElement | ImageBitmap; // RG = motion vectors
+  maxMagnitude: number;
+}
+
+/**
+ * 3D mesh data (face mesh, etc.)
+ */
+export interface MeshData {
+  type: "face-mesh";
+  vertices: Array<{ x: number; y: number; z: number }>;
+  indices: number[];
+  uvs?: Array<{ u: number; v: number }>;
+}
+
+/**
+ * Configuration value types
+ */
+export type ConfigValue = { type: "number"; min: number; max: number; default: number; label?: string } | { type: "boolean"; default: boolean; label?: string } | { type: "select"; options: string[]; default: string; label?: string } | { type: "color"; default: string; label?: string };
+
+/**
+ * Provider configuration schema
+ */
+export interface ProviderConfig {
+  [key: string]: ConfigValue;
+}
+
+/**
+ * Video frame input (can be HTMLVideoElement, canvas, or bitmap)
+ */
+export type VideoFrame = HTMLVideoElement | HTMLCanvasElement | ImageBitmap;
+
+/**
+ * Feature Provider interface
+ *
+ * All feature providers must implement this interface.
+ * Providers produce feature maps that body effects consume.
+ */
+export interface FeatureProvider {
+  /** Unique identifier */
+  id: string;
+
+  /** Human-readable name */
+  name: string;
+
+  /** Feature maps this provider produces */
+  outputs: FeatureMapType[];
+
+  /** Configuration schema */
+  config?: ProviderConfig;
+
+  /** Initialize the provider (load models, allocate resources) */
+  initialize(): Promise<void>;
+
+  /** Process a video frame and produce feature maps */
+  process(frame: VideoFrame): Promise<FeatureMap[]>;
+
+  /** Clean up resources */
+  dispose(): void;
+
+  /** Update configuration */
+  updateConfig?(config: Record<string, any>): void;
+}
+
+/**
+ * Feature provider manager
+ */
+export interface IFeatureProviderManager {
+  /** Register a provider */
+  register(provider: FeatureProvider): void;
+
+  /** Activate a provider (initialize and make ready) */
+  activate(providerId: string): Promise<void>;
+
+  /** Deactivate a provider */
+  deactivate(providerId: string): void;
+
+  /** Process frame with active providers */
+  process(frame: VideoFrame): Promise<Map<FeatureMapType, FeatureMap>>;
+
+  /** Get providers that can produce a specific feature type */
+  getProvidersForFeature(featureType: FeatureMapType): FeatureProvider[];
+
+  /** Get all registered providers */
+  getAllProviders(): FeatureProvider[];
+
+  /** Get active providers */
+  getActiveProviders(): FeatureProvider[];
+
+  /** Dispose all providers */
+  dispose(): void;
+}

package/tsconfig.json

@@ -0,0 +1,21 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "ESNext",
+    "lib": ["ES2022", "DOM"],
+    "moduleResolution": "bundler",
+    "resolveJsonModule": true,
+    "allowJs": true,
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "declaration": true,
+    "declarationMap": true,
+    "sourceMap": true,
+    "outDir": "./dist",
+    "rootDir": "./src"
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist", "**/*.test.ts"]
+}

package/tsup.config.ts

@@ -0,0 +1,16 @@
+import { defineConfig } from "tsup";
+
+export default defineConfig({
+  entry: {
+    index: "src/index.ts",
+    types: "src/types.ts",
+    "segmentation/index": "src/segmentation/index.ts",
+    "chroma-key/index": "src/chroma-key/index.ts",
+  },
+  format: ["esm"],
+  dts: true,
+  sourcemap: true,
+  clean: true,
+  splitting: false,
+  treeshake: true,
+});