@camera.ui/server 0.0.49 → 0.0.50-internal.0

package/src/camera/utils.ts

@@ -0,0 +1,489 @@
+import {
+  extractLevelFromProfileLevelId,
+  extractProfileFromProfileLevelId,
+  getAudioCodecProperties,
+  getVideoCodecProperties,
+  isSameAudioStreamInfo,
+  isSameVideoStreamInfo,
+  mapAudioCodecName,
+  mapVideoCodecName,
+  normalizeAudioCodecs,
+  normalizeVideoCodecs,
+} from '@camera.ui/common/camera';
+import type { AudioStreamInfo, RTPInfo, VideoStreamInfo } from '@camera.ui/types';
+import type { Go2RTCProducer } from '../api/go2rtc/types.js';
+
+export function generateAudioStreamInfo(producers: Go2RTCProducer[]): AudioStreamInfo[] {
+  const audioStreamInfos: AudioStreamInfo[] = [];
+
+  // Extract SDP info for additional codec details
+  const sdpInfo = extractSdpInfo(producers);
+
+  // Process all producers and collect audio info from receivers and senders
+  for (const producer of producers) {
+    // Track backchannels we've already added to avoid duplicates
+    const backchannelAdded = new Set<string>();
+
+    // Process receivers (audio streams from the device)
+    if (producer.receivers?.length) {
+      for (const receiver of producer.receivers) {
+        if (receiver.codec?.codec_type === 'audio') {
+          const audioCodec = mapAudioCodecName(receiver.codec.codec_name);
+          if (!audioCodec) continue;
+
+          const { codec, ffmpegCodec } = normalizeAudioCodecs(audioCodec);
+
+          // Create RTP info with receiver data
+          const rtpInfo: RTPInfo = {
+            codec: receiver.codec.codec_name,
+            rate: receiver.codec.sample_rate,
+            encoding: receiver.codec.channels,
+          };
+
+          // Add SDP info if available
+          const sdpCodecInfo = sdpInfo.audio[codec];
+          if (sdpCodecInfo) {
+            // Update payload if available
+            if (sdpCodecInfo.clockRate) {
+              rtpInfo.rate = sdpCodecInfo.clockRate;
+            }
+            if (sdpCodecInfo.channels) {
+              rtpInfo.encoding = sdpCodecInfo.channels;
+            }
+            if (sdpCodecInfo.payload) {
+              rtpInfo.payload = sdpCodecInfo.payload;
+            }
+          }
+
+          const properties = getAudioCodecProperties(codec, rtpInfo);
+
+          // Add fmtp parameters from SDP if available
+          if (sdpCodecInfo?.fmtp && properties.fmtpInfo) {
+            properties.fmtpInfo.config = sdpCodecInfo.fmtp;
+          }
+
+          // For MPEG4-GENERIC, add config parameter from SDP if available
+          if (codec === 'MPEG4-GENERIC' && sdpCodecInfo?.config && properties.fmtpInfo) {
+            // Füge config-Parameter zum fmtp hinzu, wenn nicht bereits enthalten
+            if (!properties.fmtpInfo.config.includes('config=')) {
+              properties.fmtpInfo.config += `;config=${sdpCodecInfo.config}`;
+            }
+          }
+
+          const audioStreamInfo: AudioStreamInfo = {
+            codec,
+            ffmpegCodec,
+            properties,
+            direction: 'sendonly', // From camera's perspective it's sending to us
+          };
+
+          // Add if unique
+          if (!audioStreamInfos.some((info) => isSameAudioStreamInfo(info, audioStreamInfo))) {
+            audioStreamInfos.push(audioStreamInfo);
+          }
+        }
+      }
+    }
+
+    // Process senders (talkback/microphone to the device)
+    if (producer.senders?.length) {
+      for (const sender of producer.senders) {
+        // Skip ANY codec as it's likely a placeholder, but we'll check medias for more info
+        if (sender.codec?.codec_type === 'audio' && sender.codec.codec_name !== 'ANY') {
+          const audioCodec = mapAudioCodecName(sender.codec.codec_name);
+          if (!audioCodec) continue;
+
+          const { codec, ffmpegCodec } = normalizeAudioCodecs(audioCodec);
+
+          const rtpInfo: RTPInfo = {
+            codec: sender.codec.codec_name,
+            rate: sender.codec.sample_rate,
+            encoding: sender.codec.channels,
+          };
+
+          const properties = getAudioCodecProperties(codec, rtpInfo);
+
+          const audioStreamInfo: AudioStreamInfo = {
+            codec,
+            ffmpegCodec,
+            properties,
+            direction: 'recvonly', // From camera's perspective it's receiving from us
+          };
+
+          // Add if unique
+          if (!audioStreamInfos.some((info) => isSameAudioStreamInfo(info, audioStreamInfo))) {
+            audioStreamInfos.push(audioStreamInfo);
+            // Mark this codec as handled for backchannel
+            backchannelAdded.add(codec);
+          }
+        }
+      }
+    }
+
+    // Check medias for backchannel if we have ANY codecs as senders or no codec info
+    if (producer.medias) {
+      for (const media of producer.medias) {
+        if (media.startsWith('audio') && media.includes('sendonly')) {
+          // This is a backchannel (talkback) from the device's perspective
+          const parts = media.split(', ');
+          if (parts.length >= 3) {
+            const codecInfo = parts[2];
+            const codecParts = codecInfo.split('/');
+            const audioCodecStr = codecParts[0];
+
+            const audioCodec = mapAudioCodecName(audioCodecStr);
+            if (!audioCodec) continue;
+
+            // Skip if we already added this codec as backchannel
+            if (backchannelAdded.has(audioCodec)) continue;
+
+            const { codec, ffmpegCodec } = normalizeAudioCodecs(audioCodec);
+
+            const rtpInfo: RTPInfo = {
+              codec: audioCodecStr,
+              rate: codecParts.length > 1 ? parseInt(codecParts[1]) : undefined,
+              encoding: codecParts.length > 2 ? parseInt(codecParts[2]) : undefined,
+            };
+
+            // Add SDP info if available
+            const sdpCodecInfo = sdpInfo.audio[codec];
+            if (sdpCodecInfo) {
+              // Update rate etc if available
+              if (sdpCodecInfo.clockRate && !rtpInfo.rate) {
+                rtpInfo.rate = sdpCodecInfo.clockRate;
+              }
+              if (sdpCodecInfo.channels && !rtpInfo.encoding) {
+                rtpInfo.encoding = sdpCodecInfo.channels;
+              }
+              if (sdpCodecInfo.payload) {
+                rtpInfo.payload = sdpCodecInfo.payload;
+              }
+            }
+
+            const properties = getAudioCodecProperties(codec, rtpInfo);
+
+            // Add fmtp parameters from SDP if available
+            if (sdpCodecInfo?.fmtp && properties.fmtpInfo) {
+              properties.fmtpInfo.config = sdpCodecInfo.fmtp;
+            }
+
+            const audioStreamInfo: AudioStreamInfo = {
+              codec,
+              ffmpegCodec,
+              properties,
+              direction: 'recvonly', // From camera's perspective it's receiving from us (talkback)
+            };
+
+            // Add if unique
+            if (!audioStreamInfos.some((info) => isSameAudioStreamInfo(info, audioStreamInfo))) {
+              audioStreamInfos.push(audioStreamInfo);
+              backchannelAdded.add(codec);
+            }
+          }
+        }
+      }
+    }
+  }
+
+  return audioStreamInfos;
+}
+
+export function generateVideoStreamInfo(producers: Go2RTCProducer[]): VideoStreamInfo[] {
+  const videoStreamInfos: VideoStreamInfo[] = [];
+
+  // Extract SDP info for additional codec details
+  const sdpInfo = extractSdpInfo(producers);
+
+  // Process all producers and collect video info from receivers
+  for (const producer of producers) {
+    // Process receivers (video streams from the device)
+    if (producer.receivers?.length) {
+      for (const receiver of producer.receivers) {
+        if (receiver.codec?.codec_type === 'video') {
+          const videoCodec = mapVideoCodecName(receiver.codec.codec_name);
+          if (!videoCodec) continue;
+
+          const { codec, ffmpegCodec } = normalizeVideoCodecs(videoCodec);
+
+          // Create RTP info
+          const rtpInfo: RTPInfo = {
+            codec: receiver.codec.codec_name,
+            rate: 90000, // Default for video
+          };
+
+          // Add SDP info if available
+          const sdpCodecInfo = sdpInfo.video[codec];
+
+          if (sdpCodecInfo) {
+            // Update payload if available
+            if (sdpCodecInfo.payload) {
+              rtpInfo.payload = sdpCodecInfo.payload;
+            }
+
+            // For H.264/H.265, extract profile and level from profile-level-id
+            if ((codec === 'H264' || codec === 'H265') && sdpCodecInfo.profileLevelId) {
+              rtpInfo.profile = extractProfileFromProfileLevelId(sdpCodecInfo.profileLevelId);
+              rtpInfo.level = extractLevelFromProfileLevelId(sdpCodecInfo.profileLevelId);
+            } else {
+              // Fallback to receiver data
+              if (receiver.codec.profile) {
+                rtpInfo.profile = receiver.codec.profile;
+              }
+              if (receiver.codec.level) {
+                rtpInfo.level = receiver.codec.level;
+              }
+            }
+          } else {
+            // Fallback to receiver data
+            if (receiver.codec.profile) {
+              rtpInfo.profile = receiver.codec.profile;
+            }
+            if (receiver.codec.level) {
+              rtpInfo.level = receiver.codec.level;
+            }
+          }
+
+          const properties = getVideoCodecProperties(codec, rtpInfo);
+
+          // Add fmtp parameters from SDP if available
+          if (sdpCodecInfo?.fmtp) {
+            if (!properties.fmtpInfo) {
+              properties.fmtpInfo = {
+                payload: rtpInfo.payload ?? properties.payloadType,
+                config: sdpCodecInfo.fmtp,
+              };
+            } else {
+              properties.fmtpInfo.config = sdpCodecInfo.fmtp;
+            }
+          } else if (codec === 'H264' && rtpInfo.profile && rtpInfo.level) {
+            // For H.264/H.265, add profile-level-id to fmtp if available
+            const profileLevelId = createProfileLevelId(rtpInfo.profile, rtpInfo.level);
+            if (profileLevelId) {
+              if (!properties.fmtpInfo) {
+                properties.fmtpInfo = {
+                  payload: rtpInfo.payload ?? properties.payloadType,
+                  config: `packetization-mode=1;profile-level-id=${profileLevelId}`,
+                };
+              } else {
+                properties.fmtpInfo.config = `packetization-mode=1;profile-level-id=${profileLevelId}`;
+              }
+            }
+          }
+
+          const videoStreamInfo: VideoStreamInfo = {
+            codec,
+            ffmpegCodec,
+            properties,
+            direction: 'sendonly', // Video is typically sendonly from camera's perspective
+          };
+
+          // Add if unique
+          if (!videoStreamInfos.some((info) => isSameVideoStreamInfo(info, videoStreamInfo))) {
+            videoStreamInfos.push(videoStreamInfo);
+          }
+        }
+      }
+    }
+  }
+
+  // If we couldn't find any video info from receivers, check medias
+  if (videoStreamInfos.length === 0) {
+    for (const producer of producers) {
+      if (producer.medias) {
+        for (const media of producer.medias) {
+          if (media.startsWith('video')) {
+            const parts = media.split(', ');
+            if (parts.length >= 3) {
+              const videoCodecStr = parts[2].split('/')[0];
+
+              const videoCodec = mapVideoCodecName(videoCodecStr);
+              if (!videoCodec) continue;
+
+              const { codec, ffmpegCodec } = normalizeVideoCodecs(videoCodec);
+
+              const rtpInfo: RTPInfo = {
+                codec: videoCodecStr,
+                rate: 90000, // Default for video
+              };
+
+              // Add SDP info if available
+              const sdpCodecInfo = sdpInfo.video[codec];
+              if (sdpCodecInfo) {
+                if (sdpCodecInfo.payload) {
+                  rtpInfo.payload = sdpCodecInfo.payload;
+                }
+
+                // For H.264/H.265, extract profile and level from profile-level-id
+                if ((codec === 'H264' || codec === 'H265') && sdpCodecInfo.profileLevelId) {
+                  rtpInfo.profile = extractProfileFromProfileLevelId(sdpCodecInfo.profileLevelId);
+                  rtpInfo.level = extractLevelFromProfileLevelId(sdpCodecInfo.profileLevelId);
+                }
+              }
+
+              const properties = getVideoCodecProperties(codec, rtpInfo);
+
+              // Add fmtp parameters from SDP if available
+              if (sdpCodecInfo?.fmtp) {
+                if (!properties.fmtpInfo) {
+                  properties.fmtpInfo = {
+                    payload: rtpInfo.payload ?? properties.payloadType,
+                    config: sdpCodecInfo.fmtp,
+                  };
+                } else {
+                  properties.fmtpInfo.config = sdpCodecInfo.fmtp;
+                }
+              }
+
+              const videoStreamInfo: VideoStreamInfo = {
+                codec,
+                ffmpegCodec,
+                properties,
+                direction: 'sendonly', // Video is typically sendonly from camera's perspective
+              };
+
+              // Add if unique
+              if (!videoStreamInfos.some((info) => isSameVideoStreamInfo(info, videoStreamInfo))) {
+                videoStreamInfos.push(videoStreamInfo);
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+
+  return videoStreamInfos;
+}
+
+export function extractSdpInfo(producers: Go2RTCProducer[]): {
+  video: Record<string, Record<string, any>>;
+  audio: Record<string, Record<string, any>>;
+} {
+  const sdpInfo = {
+    video: {} as Record<string, Record<string, any>>,
+    audio: {} as Record<string, Record<string, any>>,
+  };
+
+  for (const producer of producers) {
+    if (!producer.sdp) continue;
+
+    const sdpLines = producer.sdp.split(/\r?\n/);
+    let currentMedia: 'video' | 'audio' | null = null;
+    let currentCodec = '';
+
+    for (let i = 0; i < sdpLines.length; i++) {
+      const line = sdpLines[i].trim();
+
+      // Check for media type
+      if (line.startsWith('m=')) {
+        const mediaParts = line.split(' ');
+        if (mediaParts.length >= 4) {
+          const mediaType = mediaParts[0].substring(2) as 'audio' | 'video';
+          if (mediaType === 'audio' || mediaType === 'video') {
+            currentMedia = mediaType;
+          } else {
+            currentMedia = null;
+          }
+        }
+      } else if (currentMedia && line.startsWith('a=rtpmap:')) {
+        // Check for codec information
+        const rtpMapMatch = /a=rtpmap:(\d+)\s+([^/]+)\/(\d+)(?:\/(\d+))?/.exec(line);
+        if (rtpMapMatch) {
+          const payload = parseInt(rtpMapMatch[1], 10);
+          const codecName = rtpMapMatch[2].toUpperCase();
+          const clockRate = parseInt(rtpMapMatch[3], 10);
+          const channels = rtpMapMatch[4] ? parseInt(rtpMapMatch[4], 10) : 1;
+
+          // Initialize codec object if not present
+          if (!sdpInfo[currentMedia][codecName]) {
+            sdpInfo[currentMedia][codecName] = {};
+          }
+
+          sdpInfo[currentMedia][codecName].payload = payload;
+          sdpInfo[currentMedia][codecName].clockRate = clockRate;
+
+          if (currentMedia === 'audio') {
+            sdpInfo[currentMedia][codecName].channels = channels;
+          }
+
+          // Set current codec - we'll use this for subsequent fmtp lines
+          currentCodec = codecName;
+        }
+      } else if (currentMedia && currentCodec && line.startsWith('a=fmtp:')) {
+        // Check for fmtp information
+        const fmtpMatch = /a=fmtp:(\d+)\s+(.*)/.exec(line);
+        if (fmtpMatch) {
+          const payload = parseInt(fmtpMatch[1], 10);
+
+          // Check if this fmtp line is for the current codec
+          if (sdpInfo[currentMedia][currentCodec] && sdpInfo[currentMedia][currentCodec].payload === payload) {
+            const params = fmtpMatch[2];
+
+            // Store fmtp params
+            sdpInfo[currentMedia][currentCodec].fmtp = params;
+            sdpInfo[currentMedia][currentCodec].fmtpPayload = payload; // Store payload for reference
+
+            // For H.264/H.265, extract profile-level-id
+            if ((currentCodec === 'H264' || currentCodec === 'H265') && currentMedia === 'video') {
+              const paramMap: Record<string, string> = {};
+              const paramPairs = params.split(';');
+
+              for (const pair of paramPairs) {
+                const [key, value] = pair.split('=').map((s) => s.trim());
+                if (key && value) {
+                  paramMap[key] = value;
+                }
+              }
+
+              if (paramMap['profile-level-id']) {
+                sdpInfo[currentMedia][currentCodec].profileLevelId = paramMap['profile-level-id'];
+              }
+
+              // For H.264/H,265, extract sprop-parameter-sets, sprop-vps, sprop-sps, sprop-pps
+              ['sprop-parameter-sets', 'sprop-vps', 'sprop-sps', 'sprop-pps'].forEach((param) => {
+                if (paramMap[param]) {
+                  sdpInfo[currentMedia!][currentCodec][param] = paramMap[param];
+                }
+              });
+            }
+
+            // For MPEG4-GENERIC, extract config
+            if (currentCodec === 'MPEG4-GENERIC' && currentMedia === 'audio') {
+              const paramMap: Record<string, string> = {};
+              const paramPairs = params.split(';');
+
+              for (const pair of paramPairs) {
+                const [key, value] = pair.split('=').map((s) => s.trim());
+                if (key && value) {
+                  paramMap[key] = value;
+                }
+              }
+
+              if (paramMap.config) {
+                sdpInfo[currentMedia][currentCodec].config = paramMap.config;
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+
+  return sdpInfo;
+}
+
+export function createProfileLevelId(profile: string, level: number): string | undefined {
+  const profileMap: Record<string, string> = {
+    Baseline: '42',
+    Main: '4D',
+    High: '64',
+  };
+
+  const profileId = profileMap[profile];
+  if (!profileId) return undefined;
+
+  // Format level as hex, ensure it's 2 digits
+  const levelHex = level.toString(16).padStart(2, '0');
+
+  return `${profileId}00${levelHex}`;
+}

package/src/camera/videoFrame.ts

@@ -0,0 +1,123 @@
+import sharp from 'sharp';
+
+import type { ClientInstance } from '@camera.ui/rpc-client';
+import type { Promisify, RPCBase } from '@camera.ui/rpc-common';
+import type {
+  DecoderFormat,
+  FrameBuffer,
+  FrameData,
+  FrameImage,
+  FrameMetadata,
+  ImageInformation,
+  ImageOptions,
+  MotionSetEvent,
+  VideoFrame as VideoFrameInterface,
+} from '@camera.ui/types';
+import type { FrameWorkerInterface } from '../decoder/types.js';
+
+export class VideoFrame implements VideoFrameInterface {
+  readonly #data: Uint8Array;
+  readonly #metadata: FrameMetadata;
+  readonly #info: ImageInformation;
+  readonly #timestamp: number;
+  readonly #motion?: MotionSetEvent;
+
+  #subject: string;
+  #proxy: ClientInstance | RPCBase;
+
+  get data(): Uint8Array {
+    return this.#data;
+  }
+
+  get metadata(): FrameMetadata {
+    return this.#metadata;
+  }
+
+  get info(): ImageInformation {
+    return this.#info;
+  }
+
+  get timestamp(): number {
+    return this.#timestamp;
+  }
+
+  get motion(): MotionSetEvent | undefined {
+    return this.#motion;
+  }
+
+  get inputWidth(): number {
+    return this.#metadata.width;
+  }
+
+  get inputHeight(): number {
+    return this.#metadata.height;
+  }
+
+  get inputFormat(): DecoderFormat {
+    return this.#metadata.format;
+  }
+
+  constructor(frameData: FrameData, subject: string, proxy: ClientInstance | RPCBase, motion?: MotionSetEvent) {
+    this.#data = frameData.data;
+    this.#metadata = frameData.metadata;
+    this.#info = frameData.info;
+    this.#timestamp = frameData.timestamp;
+    this.#motion = motion;
+    this.#subject = subject;
+    this.#proxy = proxy;
+  }
+
+  public async toBuffer(options: ImageOptions = {}): Promise<FrameBuffer> {
+    let processedFrame: FrameBuffer = {
+      image: this.data,
+      info: this.info,
+    };
+
+    if (Object.keys(options).length > 0) {
+      options = this.#validateOptions('buffer', options);
+      const frameWorkerProxy = await this.#frameWorkerProxy();
+      processedFrame = await frameWorkerProxy.processFrame(this.data, this.inputWidth, this.inputHeight, this.inputFormat, options);
+    }
+
+    return {
+      image: Buffer.from(processedFrame.image.buffer, processedFrame.image.byteOffset, processedFrame.image.byteLength),
+      info: processedFrame.info,
+    };
+  }
+
+  public async toImage(options: ImageOptions = {}): Promise<FrameImage> {
+    let processedFrame: FrameBuffer = {
+      image: this.data,
+      info: this.info,
+    };
+
+    if (Object.keys(options).length > 0) {
+      options = this.#validateOptions('image', options);
+      const frameWorkerProxy = await this.#frameWorkerProxy();
+      processedFrame = await frameWorkerProxy.processFrame(this.data, this.inputWidth, this.inputHeight, this.inputFormat, options);
+    }
+
+    return {
+      image: sharp(processedFrame.image, { raw: processedFrame.info as any }),
+      info: processedFrame.info,
+    };
+  }
+
+  public async save(path: string, options: ImageOptions = {}): Promise<void> {
+    const { image } = await this.toImage(options);
+    await image.toFile(path);
+    image.destroy();
+  }
+
+  async #frameWorkerProxy(): Promise<Promisify<FrameWorkerInterface>> {
+    return await this.#proxy.createRPCProxy<FrameWorkerInterface>(this.#subject);
+  }
+
+  #validateOptions(target: 'buffer' | 'image', options: ImageOptions = {}): ImageOptions {
+    if (!options.format && target === 'image') {
+      options.format = { to: 'rgb' };
+    }
+
+    return options;
+  }
+}

package/src/cloud/index.ts

@@ -0,0 +1,61 @@
+import { container } from 'tsyringe';
+
+import { CloudflareService } from './remote/cloudflare.js';
+import { UPNPService } from './remote/upnp.js';
+
+import type { Database } from '../api/database/index.js';
+import type { ConfigService } from '../services/config/index.js';
+import type { LoggerService } from '../services/logger/index.js';
+
+export class CloudService {
+  public readonly cloudflare: CloudflareService;
+  public readonly upnp: UPNPService;
+
+  private dbs: Database;
+  private logger: LoggerService;
+  private configService: ConfigService;
+
+  constructor() {
+    container.registerInstance('cloud', this);
+
+    this.dbs = container.resolve<Database>('dbs');
+    this.logger = container.resolve<LoggerService>('logger');
+    this.configService = container.resolve<ConfigService>('configService');
+
+    this.cloudflare = new CloudflareService();
+    this.upnp = new UPNPService();
+  }
+
+  public async initialize(): Promise<void> {
+    const systemConfig = this.dbs.systemDB.get('system');
+    if (systemConfig) {
+      const mode = systemConfig.remoteAccess.mode;
+
+      switch (mode) {
+        case 'disabled':
+          this.logger.log('Remote access is disabled');
+          await this.cloudflare.stop();
+          await this.upnp.stop();
+          break;
+        case 'cloudflare':
+          this.logger.log('Remote access is enabled with Cloudflare');
+          await this.cloudflare.start();
+          break;
+        case 'upnp':
+          this.logger.log('Remote access is enabled with UPnP');
+          await this.upnp.start();
+          break;
+        case 'port-forward':
+        case 'router-forward':
+          this.logger.log('Remote access is enabled with port/router forwarding');
+          await this.upnp.start();
+          break;
+        case 'custom-domain':
+          this.logger.log('Remote access is enabled with custom domain');
+          break;
+      }
+    }
+  }
+
+  public async stop(): Promise<void> {}
+}

package/src/cloud/remote/cloudflare.ts

@@ -0,0 +1,18 @@
+import { container } from 'tsyringe';
+
+import type { Database } from '../../api/database/index.js';
+import type { SocketService } from '../../api/websocket/index.js';
+
+export class CloudflareService {
+  private dbs: Database;
+  private socketService: SocketService;
+
+  constructor() {
+    this.dbs = container.resolve<Database>('dbs');
+    this.socketService = container.resolve<SocketService>('socket');
+  }
+
+  public async start(): Promise<void> {}
+
+  public async stop(): Promise<void> {}
+}