@camera.ui/server 0.0.49 → 0.0.50-internal.0

package/src/camera/device.ts

@@ -0,0 +1,422 @@
+import { buildTargetUrl } from '@camera.ui/common/camera';
+import { CameraDeviceCapability } from '@camera.ui/types';
+
+import { BaseCameraDevice } from './index.js';
+import { FMP4Session } from './streaming/fmp4-session.js';
+import { RTSPSession } from './streaming/rtsp-session.js';
+import { WeriftSession } from './streaming/werift-session.js';
+import { VideoFrame } from './videoFrame.js';
+
+import type { Message, Promisify, RPCBase, SubscriptionWrapper } from '@camera.ui/rpc-common';
+import type {
+  AudioSetEvent,
+  BatterySetEvent,
+  Camera,
+  CameraDevice as CameraDeviceInterface,
+  CameraDeviceSource,
+  CameraInput,
+  DoorbellSetEvent,
+  FMP4ConnectionOptions,
+  FrameData,
+  FrameType,
+  ImageOptions,
+  LightSetEvent,
+  LoggerService,
+  MotionDetectionFrame,
+  MotionSetEvent,
+  MotionState,
+  ObjectSetEvent,
+  ProbeConfig,
+  ProbeStream,
+  PTZInterface,
+  RTSPConnectionOptions,
+  RTSPUrlOptions,
+  SirenSetEvent,
+  SnapshotInterface,
+  StreamingInterface,
+  WebRTCConnectionOptions,
+} from '@camera.ui/types';
+import type { Subscription } from 'rxjs';
+import type { FrameWorkerInterface } from '../decoder/types.js';
+import type { CameraNamespaces, FrameWorkerNamespaces } from '../rpc/namespaces.js';
+
+export abstract class CameraDevice extends BaseCameraDevice implements CameraDeviceInterface {
+  get sources(): CameraDeviceSource[] {
+    const sources: CameraInput[] = JSON.parse(JSON.stringify(this.cameraSubject.getValue().sources));
+
+    return sources.map((source): CameraDeviceSource => {
+      return {
+        ...source,
+        generateRTSPUrl: (options: RTSPUrlOptions): string => {
+          return buildTargetUrl(source.urls.rtsp.base, options);
+        },
+        snapshot: (forceNew): Promise<ArrayBuffer | undefined> => {
+          return this.snapshot(source._id, forceNew);
+        },
+        probeStream: (probeConfig?: ProbeConfig, refresh = false): Promise<ProbeStream | undefined> => {
+          return this.probeStream(source._id, probeConfig, refresh);
+        },
+        createWebRTCSession: (options?: WebRTCConnectionOptions): WeriftSession => {
+          return new WeriftSession(this, source, options);
+        },
+        createRTSPSession: (options?: RTSPConnectionOptions): RTSPSession => {
+          return new RTSPSession(this, source, options);
+        },
+        createFMP4Session: (options?: FMP4ConnectionOptions): FMP4Session => {
+          return new FMP4Session(this, source, options);
+        },
+      };
+    });
+  }
+
+  get streamSource(): CameraDeviceSource {
+    return (this.highResolutionSource ?? this.midResolutionSource ?? this.lowResolutionSource)!;
+  }
+
+  get highResolutionSource(): CameraDeviceSource | undefined {
+    return this.sources.find((source) => source.role === 'high-resolution');
+  }
+
+  get midResolutionSource(): CameraDeviceSource | undefined {
+    return this.sources.find((source) => source.role === 'mid-resolution');
+  }
+
+  get lowResolutionSource(): CameraDeviceSource | undefined {
+    return this.sources.find((source) => source.role === 'low-resolution');
+  }
+
+  #proxy: RPCBase;
+  #namespaces: CameraNamespaces & FrameWorkerNamespaces;
+
+  protected abstract initialized: boolean;
+
+  constructor(camera: Camera, proxy: RPCBase, namespaces: CameraNamespaces & FrameWorkerNamespaces, logger: LoggerService) {
+    super(camera, logger);
+    this.#proxy = proxy;
+    this.#namespaces = namespaces;
+  }
+
+  public getFrames(type: 'stream', options?: ImageOptions): AsyncIterableIterator<VideoFrame>;
+  public getFrames(type: 'motion', options?: ImageOptions): AsyncIterableIterator<VideoFrame>;
+  public async *getFrames(type: FrameType, options?: ImageOptions): AsyncIterableIterator<VideoFrame> {
+    let subscription: SubscriptionWrapper | undefined;
+    let capSubscription: Subscription | undefined;
+    let motionSubscription: Subscription | undefined;
+    let currentResolver: ((value: MotionDetectionFrame | undefined) => void) | undefined;
+    let currentType = type;
+
+    // For motion sensor only
+    let motionDetected = false;
+    let frameWorkerGenerator: AsyncIterableIterator<FrameData> | undefined;
+    let pendingFramePromise: Promise<IteratorResult<FrameData>> | undefined;
+
+    const isMotionDetector = () => {
+      return this.hasMotionDetector && this.hasCapability(CameraDeviceCapability.MotionDetector);
+    };
+
+    const isMotionSensorOnly = () => {
+      return this.hasMotionDetector && !this.hasCapability(CameraDeviceCapability.MotionDetector) && this.hasCapability(CameraDeviceCapability.MotionSensor);
+    };
+
+    const stopFrameWorkerGenerator = async () => {
+      if (frameWorkerGenerator) {
+        try {
+          await frameWorkerGenerator.return?.();
+        } catch {
+          //
+        }
+
+        frameWorkerGenerator = undefined;
+        pendingFramePromise = undefined;
+      }
+    };
+
+    try {
+      const frameWorkerProxy = await this.#frameWorkerProxy();
+      const queue: MotionDetectionFrame[] = [];
+
+      // Subscribe to motion frames if needed for motion detector
+      if (currentType === 'motion') {
+        const onFrame = (message: Message<MotionDetectionFrame>) => {
+          if (currentResolver) {
+            currentResolver(message.data);
+            currentResolver = undefined;
+          } else {
+            queue.push(message.data);
+          }
+        };
+
+        subscription = this.#proxy.subscribe<MotionDetectionFrame>(this.#namespaces.cameraMotionFrameSubject, onFrame);
+
+        // For motion sensor, subscribe to motion events
+        if (isMotionSensorOnly()) {
+          // Initialize motion state from current value
+          motionDetected = this.motionState.getValue().state || false;
+
+          // Subscribe to motion state changes
+          motionSubscription = this.onMotionDetected.subscribe((state: MotionState) => {
+            const newMotionState = state.state ?? false;
+            if (newMotionState !== motionDetected) {
+              motionDetected = newMotionState;
+
+              // If motion just started and we're waiting, resolve to continue the loop
+              if (motionDetected && currentResolver) {
+                currentResolver(undefined);
+                currentResolver = undefined;
+              }
+            }
+          });
+        }
+      }
+
+      // Subscribe to changes in capabilities
+      if (type === 'motion') {
+        capSubscription = this.onCapabilitiesChange().subscribe(async ({ newCapabilities, oldCapabilities }) => {
+          const hadMotionDetector = oldCapabilities.includes(CameraDeviceCapability.MotionDetector);
+          const hasMotionDetectorNow = newCapabilities.includes(CameraDeviceCapability.MotionDetector);
+          const hasNewMotionDetector = isMotionDetector() || isMotionSensorOnly();
+
+          if (currentType === 'motion' && !hasNewMotionDetector) {
+            // Case 1: Currently in motion mode, but no motion detector available anymore
+            this.logger.trace('Switching to stream mode due to missing motion detector');
+
+            // Reset any motion sensor state
+            await stopFrameWorkerGenerator();
+
+            // If we're waiting for motion, resolve to restart the loop
+            if (currentResolver) {
+              currentResolver(undefined);
+              currentResolver = undefined;
+            }
+          } else if (currentType === 'stream' && hasNewMotionDetector) {
+            // Case 2: Currently in stream mode, but motion detector available now
+            this.logger.trace('Switching to motion mode due to motion detector being available');
+            currentType = type;
+          } else if (currentType === 'motion' && !hadMotionDetector && hasMotionDetectorNow) {
+            // Case 3: Currently in motion sensor mode, but motion detector available now
+            this.logger.trace('Switching from motion sensor to motion detector');
+            // Reset any motion sensor state
+            await stopFrameWorkerGenerator();
+
+            // If we're waiting for motion, resolve to restart the loop
+            if (currentResolver) {
+              currentResolver(undefined);
+              currentResolver = undefined;
+            }
+          } else if (currentType === 'motion' && hadMotionDetector && !hasMotionDetectorNow && isMotionSensorOnly()) {
+            // Case 4: Currently using motion detector, but now switching back to motion sensor
+            this.logger.trace('Switching from motion detector to motion sensor');
+
+            // If we're waiting for motion, resolve to restart the loop
+            if (currentResolver) {
+              currentResolver(undefined);
+              currentResolver = undefined;
+            }
+          }
+        });
+      }
+
+      const getPrebufferedFrame = async (timestamp: number): Promise<FrameData | undefined> => {
+        try {
+          return frameWorkerProxy.getPrebufferedFrame(timestamp, options);
+        } catch {
+          //
+        }
+      };
+
+      const processMotionFrame = async (motionFrame: MotionDetectionFrame): Promise<FrameData> => {
+        const frameFormat = motionFrame.frame.info.format;
+        const frameWidth = motionFrame.frame.info.width;
+        const frameHeight = motionFrame.frame.info.height;
+
+        const requestedFormat = options?.format?.to ?? frameFormat;
+        const requestedWidth = options?.resize?.width ?? options?.crop?.width ?? frameWidth;
+        const requestedHeight = options?.resize?.height ?? options?.crop?.height ?? frameHeight;
+
+        const processFrame = requestedFormat !== frameFormat || requestedWidth !== frameWidth || requestedHeight !== frameHeight;
+
+        if (processFrame && options) {
+          const prebufferedFrame = await getPrebufferedFrame(motionFrame.frame.timestamp);
+          if (prebufferedFrame) {
+            return prebufferedFrame;
+          } else {
+            const format = frameFormat === 'rgb' ? 'rgb24' : frameFormat;
+            const frameData = await frameWorkerProxy.processFrame(motionFrame.frame.data, frameWidth, frameHeight, format, options);
+            return {
+              data: frameData.image,
+              info: frameData.info,
+              metadata: {
+                format: motionFrame.frame.metadata.format,
+                frameSize: motionFrame.frame.metadata.frameSize,
+                height: frameData.info.height,
+                width: frameData.info.width,
+                origHeight: motionFrame.frame.info.height,
+                origWidth: motionFrame.frame.info.width,
+              },
+              timestamp: motionFrame.frame.timestamp,
+            };
+          }
+        } else {
+          return motionFrame.frame;
+        }
+      };
+
+      while (this.initialized) {
+        // Handle motion detector
+        if (currentType === 'motion' && isMotionDetector()) {
+          if (queue.length > 0) {
+            const motionFrame = queue.shift()!;
+            motionFrame.frame = await processMotionFrame(motionFrame);
+            yield new VideoFrame(motionFrame.frame, this.#namespaces.frameWorkerChildRpc, this.#proxy, motionFrame.state);
+          } else {
+            const motionFrame = await new Promise<MotionDetectionFrame | undefined>((resolve) => {
+              currentResolver = resolve;
+            });
+            if (!motionFrame) {
+              currentType = 'stream';
+              continue;
+            }
+            motionFrame.frame = await processMotionFrame(motionFrame);
+            yield new VideoFrame(motionFrame.frame, this.#namespaces.frameWorkerChildRpc, this.#proxy, motionFrame.state);
+          }
+        } else if (currentType === 'motion' && isMotionSensorOnly()) {
+          // Handle motion sensor
+
+          if (!frameWorkerGenerator) {
+            this.logger.trace('Initializing frame worker generator for motion sensor');
+          }
+
+          // Initialize frame generator if needed
+          frameWorkerGenerator ??= frameWorkerProxy.generateFrames({ isGenerator: true }, options);
+
+          // If no motion detected, wait for it
+          if (!motionDetected) {
+            await new Promise<void>((resolve) => {
+              currentResolver = () => {
+                resolve();
+                return undefined as any;
+              };
+            });
+            continue;
+          }
+
+          // Get next frame (create promise if needed)
+          pendingFramePromise ??= frameWorkerGenerator.next();
+
+          // Wait for frame
+          const frameResult = await pendingFramePromise;
+          pendingFramePromise = undefined;
+
+          if (frameResult.done) {
+            // Generator ended, reset and switch to stream
+            frameWorkerGenerator = undefined;
+            currentType = 'stream';
+            continue;
+          }
+
+          // Queue up next frame request (but don't await it yet)
+          pendingFramePromise = frameWorkerGenerator.next();
+
+          // Create a synthetic motion state for the frame
+          const motionState: MotionSetEvent = {
+            state: true,
+            detections: this.motionState.getValue().detections ?? [],
+          };
+
+          // Yield the frame with motion state
+          yield new VideoFrame(frameResult.value, this.#namespaces.frameWorkerChildRpc, this.#proxy, motionState);
+        } else {
+          // Handle regular streaming
+
+          // Reset motion sensor resources if switching modes
+          if (frameWorkerGenerator) {
+            frameWorkerGenerator = undefined;
+            pendingFramePromise = undefined;
+          }
+
+          // Regular stream mode
+          for await (const frame of frameWorkerProxy.generateFrames({ isGenerator: true }, options)) {
+            if ((currentType === 'motion' && isMotionDetector()) || (currentType === 'motion' && isMotionSensorOnly())) {
+              break; // Break stream loop to switch back to motion mode
+            }
+            yield new VideoFrame(frame, this.#namespaces.frameWorkerChildRpc, this.#proxy);
+          }
+        }
+      }
+    } catch (error) {
+      if (!this.initialized) {
+        return;
+      }
+      throw error;
+    } finally {
+      subscription?.unsubscribe();
+      capSubscription?.unsubscribe();
+      motionSubscription?.unsubscribe();
+    }
+  }
+
+  public abstract addCapabilities(capabilities: CameraDeviceCapability[]): void;
+  public abstract removeCapabilities(capabilities: CameraDeviceCapability[]): void;
+  public abstract extend(interfaces: PTZInterface | SnapshotInterface | StreamingInterface): void;
+
+  public abstract connect(): Promise<void>;
+  public abstract disconnect(): Promise<void>;
+
+  public abstract updateState(stateName: 'light', eventData: LightSetEvent): Promise<void>;
+  public abstract updateState(stateName: 'motion', eventData: MotionSetEvent, frame?: VideoFrame): Promise<void>;
+  public abstract updateState(stateName: 'audio', eventData: AudioSetEvent): Promise<void>;
+  public abstract updateState(stateName: 'object', eventData: ObjectSetEvent): Promise<void>;
+  public abstract updateState(stateName: 'doorbell', eventData: DoorbellSetEvent): Promise<void>;
+  public abstract updateState(stateName: 'siren', eventData: SirenSetEvent): Promise<void>;
+  public abstract updateState(stateName: 'battery', eventData: BatterySetEvent): Promise<void>;
+
+  protected abstract snapshot(sourceId: string, forceNew?: boolean): Promise<ArrayBuffer | undefined>;
+  protected abstract probeStream(sourceId: string, probeConfig?: ProbeConfig, refresh?: boolean): Promise<ProbeStream | undefined>;
+
+  protected addCapabilitiesInternal(pluginId: string, capabilities: CameraDeviceCapability[]): boolean {
+    const newCapabilities: CameraDeviceCapability[] = [];
+
+    for (const capability of capabilities) {
+      if (!this.hasCapability(capability)) {
+        newCapabilities.push(capability);
+      }
+    }
+
+    if (newCapabilities.length > 0) {
+      const allCapabilities = structuredClone(this.capabilities.getValue());
+      const pluginRelatedCapabilities = allCapabilities.find((cap) => cap.pluginId === pluginId);
+
+      if (pluginRelatedCapabilities) {
+        pluginRelatedCapabilities.capabilities.push(...newCapabilities);
+      } else {
+        allCapabilities.push({ pluginId, capabilities: newCapabilities });
+      }
+
+      this.capabilities.next(allCapabilities);
+      return true;
+    }
+
+    return false;
+  }
+
+  protected removeCapabilitiesInternal(pluginId: string, capabilities: CameraDeviceCapability[]): boolean {
+    let allCapabilities = structuredClone(this.capabilities.getValue());
+
+    const pluginRelatedCapabilities = allCapabilities.find((cap) => cap.pluginId === pluginId);
+    if (pluginRelatedCapabilities) {
+      pluginRelatedCapabilities.capabilities = capabilities.length === 0 ? [] : pluginRelatedCapabilities.capabilities.filter((cap) => !capabilities.includes(cap));
+      if (pluginRelatedCapabilities.capabilities.length === 0) {
+        allCapabilities = allCapabilities.filter((cap) => cap.pluginId !== pluginId);
+      }
+
+      this.capabilities.next(allCapabilities);
+      return true;
+    }
+
+    return false;
+  }
+
+  async #frameWorkerProxy(): Promise<Promisify<FrameWorkerInterface>> {
+    return await this.#proxy.createRPCProxy<FrameWorkerInterface>(this.#namespaces.frameWorkerChildRpc);
+  }
+}