@camstack/server 0.1.3

package/src/__tests__/streaming-diagnostic.test.ts

@@ -0,0 +1,512 @@
+/* eslint-disable @typescript-eslint/no-unsafe-argument -- test file, mock typing */
+/**
+ * Stream diagnostic — measures quality, latency, and fanout.
+ * Run: FRIGATE_HOST=192.168.1.128 npx vitest run server/backend/src/__tests__/streaming-diagnostic.test.ts --reporter verbose
+ */
+
+// E2E diagnostic test with heavy broker usage — file-level disable for cascading unsafe rules
+/* eslint-disable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-explicit-any */
+
+import { describe, it, expect, afterAll } from 'vitest'
+import { StreamBrokerManager, FfmpegDecoderProvider } from '@camstack/addon-pipeline/stream-broker'
+import type { EncodedPacket, FrameHandle, IStreamBroker } from '@camstack/types'
+import type { IScopedLogger } from '@camstack/types'
+
+const FRIGATE = process.env.FRIGATE_HOST ?? ''
+const STREAMS = [
+  { id: 'ingresso_main', url: `rtsp://${FRIGATE}:8554/ingresso_main`, codec: 'h264' },
+  { id: 'salone_main', url: `rtsp://${FRIGATE}:8554/salone_main`, codec: 'h264' },
+]
+
+const mockLogger: IScopedLogger = {
+  debug: () => {},
+  info: () => {},
+  warn: console.warn,
+  error: console.error,
+  child: () => mockLogger,
+}
+
+const mockLoggingService = {
+  createLogger: () => mockLogger,
+}
+
+function wait(ms: number) {
+  return new Promise(r => setTimeout(r, ms))
+}
+
+function waitForFrames(count: number, frames: unknown[], timeoutMs: number): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const start = Date.now()
+    const iv = setInterval(() => {
+      if (frames.length >= count) { clearInterval(iv); resolve() }
+      if (Date.now() - start > timeoutMs) { clearInterval(iv); reject(new Error(`Timeout: got ${frames.length}/${count} frames`)) }
+    }, 50)
+  })
+}
+
+/**
+ * Poll-based replacement for the removed `IStreamBroker.onDecodedFrame`
+ * callback (Phase 5 / D9 — decoded frames now travel as shm `FrameHandle`s).
+ * Opens a frame-handle subscription, drains `pullFrameHandles` on a 50ms
+ * timer, and invokes `onHandle` once per decoded frame. The returned
+ * function tears the subscription down. `byteLength` stands in for the old
+ * `DecodedFrame.data.length`; `pts` for `DecodedFrame.timestamp`.
+ */
+async function subscribeDecodedFrames(
+  broker: IStreamBroker,
+  maxFps: number,
+  onHandle: (handle: FrameHandle) => void,
+): Promise<() => Promise<void>> {
+  const { subscriptionId } = await broker.subscribeFrameHandles({ format: 'rgb', maxFps })
+  const iv = setInterval(() => {
+    for (const handle of broker.pullFrameHandles(subscriptionId, 8)) {
+      onHandle(handle)
+    }
+  }, 50)
+  return async () => {
+    clearInterval(iv)
+    await broker.unsubscribeFrameHandles(subscriptionId)
+  }
+}
+
+describe.skipIf(!FRIGATE)('Stream Diagnostic (requires Frigate)', () => {
+  const manager = new StreamBrokerManager([new FfmpegDecoderProvider()], mockLogger)
+
+  afterAll(async () => {
+    await manager.destroyAll()
+  })
+
+  it('measures first-frame latency per stream', async () => {
+    console.log('\n--- First-Frame Latency ---')
+
+    for (const s of STREAMS) {
+      const broker = await manager.createBroker(`latency-${s.id}`, {
+        type: 'rtsp', url: s.url, videoCodec: s.codec,
+      })
+      const startMs = Date.now()
+      await (broker as any).start({ type: 'rtsp', url: s.url, videoCodec: s.codec })
+
+      const result = await new Promise<{ latencyMs: number; sizeKB: number }>((resolve, reject) => {
+        const timeout = setTimeout(() => reject(new Error('No frame in 15s')), 15000)
+        let unsub: (() => Promise<void>) | undefined
+        subscribeDecodedFrames(broker, 10, (handle: FrameHandle) => {
+          clearTimeout(timeout)
+          void unsub?.()
+          resolve({ latencyMs: Date.now() - startMs, sizeKB: +(handle.byteLength / 1024).toFixed(1) })
+        }).then((fn) => { unsub = fn }).catch(reject)
+      })
+
+      console.log(`  ${s.id}: ${result.latencyMs}ms, frame=${result.sizeKB}KB`)
+      expect(result.latencyMs).toBeLessThan(15000)
+      expect(result.sizeKB).toBeGreaterThan(1) // at least 1KB JPEG
+
+      await broker.stop()
+      await manager.destroyBroker(`latency-${s.id}`)
+    }
+  }, 30000)
+
+  it('measures frame quality over 5 seconds', async () => {
+    console.log('\n--- Frame Quality (5s @ 5fps) ---')
+
+    const broker = await manager.createBroker('quality', {
+      type: 'rtsp', url: STREAMS[0]!.url, videoCodec: 'h264',
+    })
+    await (broker as any).start({ type: 'rtsp', url: STREAMS[0]!.url, videoCodec: 'h264' })
+
+    const frames: { sizeKB: number; ts: number }[] = []
+
+    // Wait for first frame before measuring
+    await new Promise<void>((resolve, reject) => {
+      const timeout = setTimeout(() => reject(new Error('No initial frame')), 10000)
+      let unsub: (() => Promise<void>) | undefined
+      subscribeDecodedFrames(broker, 5, () => {
+        clearTimeout(timeout)
+        void unsub?.()
+        resolve()
+      }).then((fn) => { unsub = fn }).catch(reject)
+    })
+
+    // Now collect for 5 seconds
+    const unsub = await subscribeDecodedFrames(broker, 5, (handle: FrameHandle) => {
+      frames.push({ sizeKB: +(handle.byteLength / 1024).toFixed(1), ts: handle.pts })
+    })
+
+    await wait(5000)
+    await unsub()
+
+    const sizes = frames.map(f => f.sizeKB)
+    const avgSize = +(sizes.reduce((a, b) => a + b, 0) / sizes.length).toFixed(1)
+    const minSize = Math.min(...sizes)
+    const maxSize = Math.max(...sizes)
+
+    // Frame-to-frame gaps
+    const gaps: number[] = []
+    for (let i = 1; i < frames.length; i++) {
+      gaps.push(frames[i]!.ts - frames[i - 1]!.ts)
+    }
+    const avgGap = gaps.length > 0 ? Math.round(gaps.reduce((a, b) => a + b, 0) / gaps.length) : 0
+    const maxGap = gaps.length > 0 ? Math.max(...gaps) : 0
+
+    console.log(`  Frames: ${frames.length} (expected ~25 @ 5fps)`)
+    console.log(`  JPEG size: avg=${avgSize}KB, min=${minSize}KB, max=${maxSize}KB`)
+    console.log(`  Frame gap: avg=${avgGap}ms, max=${maxGap}ms (expected ~200ms @ 5fps)`)
+    console.log(`  All valid: ${frames.every(f => f.sizeKB > 0.5) ? 'YES' : 'NO'}`)
+
+    expect(frames.length).toBeGreaterThan(10)
+    expect(avgSize).toBeGreaterThan(1) // > 1KB avg = real JPEG content
+    expect(avgGap).toBeLessThan(500) // not more than 500ms avg gap at 5fps
+
+    await broker.stop()
+    await manager.destroyBroker('quality')
+  }, 30000)
+
+  it('measures fanout: encoded + 2 decoded subscribers', async () => {
+    console.log('\n--- Fanout (3s, encoded + 2 decoded) ---')
+
+    const broker = await manager.createBroker('fanout', {
+      type: 'rtsp', url: STREAMS[0]!.url, videoCodec: 'h264',
+    })
+    await (broker as any).start({ type: 'rtsp', url: STREAMS[0]!.url, videoCodec: 'h264' })
+
+    let encodedCount = 0
+    let encodedBytes = 0
+    let keyframes = 0
+    let decodedA = 0
+    let decodedB = 0
+
+    // Wait for stream to start
+    await new Promise<void>((resolve, reject) => {
+      const timeout = setTimeout(() => reject(new Error('No frame')), 10000)
+      let unsub: (() => Promise<void>) | undefined
+      subscribeDecodedFrames(broker, 10, () => {
+        clearTimeout(timeout)
+        void unsub?.()
+        resolve()
+      }).then((fn) => { unsub = fn }).catch(reject)
+    })
+
+    // Subscribe all 3
+    const unsubE = broker.onEncodedData((pkt: EncodedPacket) => {
+      encodedCount++
+      encodedBytes += pkt.data.length
+      if (pkt.keyframe) keyframes++
+    })
+    // Phase 5 / D9: the broker no longer throttles per-subscriber fps — the
+    // shm frame plane keeps one decoder session per format (here both
+    // subscribers share the `rgb` ring) and latest-wins ring reads drop
+    // frames implicitly. Both subscribers therefore see the same cadence;
+    // the old per-subscriber fps-isolation guarantee is gone.
+    const unsubD1 = await subscribeDecodedFrames(broker, 5, () => { decodedA++ })
+    const unsubD2 = await subscribeDecodedFrames(broker, 5, () => { decodedB++ })
+
+    await wait(3000)
+
+    // Capture stats BEFORE unsubscribing
+    const stats = broker.getStats()
+
+    unsubE()
+    await unsubD1()
+    await unsubD2()
+
+    console.log(`  Encoded packets: ${encodedCount} (${(encodedBytes / 1024).toFixed(0)}KB, ${keyframes} keyframes)`)
+    console.log(`  Decoded subscriber A: ${decodedA} frames`)
+    console.log(`  Decoded subscriber B: ${decodedB} frames`)
+    console.log(`  Broker stats: inputFps=${stats.inputFps}, decodeFps=${stats.decodeFps}, uptime=${stats.uptimeMs}ms`)
+
+    // Assertions
+    expect(encodedCount).toBeGreaterThan(0)
+    expect(decodedA).toBeGreaterThan(0)
+    expect(decodedB).toBeGreaterThan(0)
+    expect(stats.decodeFps).toBeGreaterThan(0) // plane's aggregate decode rate
+    expect(keyframes).toBeGreaterThan(0) // at least 1 keyframe in 3s
+
+    await broker.stop()
+    await manager.destroyBroker('fanout')
+  }, 30000)
+
+  it('measures concurrent broker isolation (2 cameras)', async () => {
+    console.log('\n--- Concurrent Brokers (2 cameras, 3s @ 3fps) ---')
+
+    const results: Record<string, { frames: number; sizes: number[] }> = {}
+    const brokerIds: string[] = []
+    const collectUnsubs: Array<() => Promise<void>> = []
+
+    for (const s of STREAMS) {
+      const brokerId = `concurrent-${s.id}`
+      brokerIds.push(brokerId)
+      const broker = await manager.createBroker(brokerId, {
+        type: 'rtsp', url: s.url, videoCodec: s.codec,
+      })
+      await (broker as any).start({ type: 'rtsp', url: s.url, videoCodec: s.codec })
+      results[s.id] = { frames: 0, sizes: [] }
+
+      // Wait for first frame
+      await new Promise<void>((resolve, reject) => {
+        const timeout = setTimeout(() => reject(new Error(`No frame from ${s.id}`)), 10000)
+        let unsub: (() => Promise<void>) | undefined
+        subscribeDecodedFrames(broker, 5, () => {
+          clearTimeout(timeout)
+          void unsub?.()
+          resolve()
+        }).then((fn) => { unsub = fn }).catch(reject)
+      })
+
+      collectUnsubs.push(await subscribeDecodedFrames(broker, 3, (handle: FrameHandle) => {
+        results[s.id]!.frames++
+        results[s.id]!.sizes.push(handle.byteLength)
+      }))
+    }
+
+    await wait(3000)
+    for (const unsub of collectUnsubs) await unsub()
+
+    for (const s of STREAMS) {
+      const r = results[s.id]!
+      const avgKB = r.sizes.length > 0 ? +(r.sizes.reduce((a, b) => a + b, 0) / r.sizes.length / 1024).toFixed(1) : 0
+      console.log(`  ${s.id}: ${r.frames} frames, avg=${avgKB}KB`)
+    }
+
+    const allProducing = Object.values(results).every(r => r.frames > 0)
+    console.log(`  All producing: ${allProducing ? 'YES' : 'FAIL'}`)
+
+    expect(allProducing).toBe(true)
+
+    for (const id of brokerIds) {
+      const b = manager.getBroker(id)
+      if (b) { await b.stop() }
+      await manager.destroyBroker(id)
+    }
+  }, 30000)
+
+  it('stress test: 3 cameras x 3 subscribers each at different FPS', async () => {
+    console.log('\n--- Stress: 3 cameras x 3 subs (5s) ---')
+
+    const cameras = [
+      { id: 'ingresso_main', url: `rtsp://${FRIGATE}:8554/ingresso_main` },
+      { id: 'ingresso_sub', url: `rtsp://${FRIGATE}:8554/ingresso_sub` },
+      { id: 'salone_main', url: `rtsp://${FRIGATE}:8554/salone_main` },
+    ]
+
+    // Phase 5 / D9: the broker no longer throttles per-subscriber fps. This
+    // stays a load test — 3 concurrent frame-handle subscribers per camera —
+    // but the obsolete per-subscriber fps-isolation assertion is dropped.
+    const subscriberCount = 3
+    const results: Record<string, Record<number, { frames: number; sizes: number[] }>> = {}
+    const unsubs: Array<() => Promise<void>> = []
+    const brokerIds: string[] = []
+
+    // Start all 3 cameras
+    for (const cam of cameras) {
+      const brokerId = `stress-${cam.id}`
+      brokerIds.push(brokerId)
+      results[cam.id] = {}
+
+      const broker = await manager.createBroker(brokerId, {
+        type: 'rtsp', url: cam.url, videoCodec: 'h264',
+      })
+      await (broker as any).start({ type: 'rtsp', url: cam.url, videoCodec: 'h264' })
+
+      // Wait for first frame (shared decoder warmup)
+      await new Promise<void>((resolve, reject) => {
+        const timeout = setTimeout(() => reject(new Error(`No frame from ${cam.id} in 10s`)), 10000)
+        let unsub: (() => Promise<void>) | undefined
+        subscribeDecodedFrames(broker, 10, () => {
+          clearTimeout(timeout)
+          void unsub?.()
+          resolve()
+        }).then((fn) => { unsub = fn }).catch(reject)
+      })
+
+      // Subscribe N concurrent frame-handle readers
+      for (let sub = 0; sub < subscriberCount; sub++) {
+        results[cam.id]![sub] = { frames: 0, sizes: [] }
+        unsubs.push(await subscribeDecodedFrames(broker, 5, (handle: FrameHandle) => {
+          results[cam.id]![sub]!.frames++
+          results[cam.id]![sub]!.sizes.push(handle.byteLength)
+        }))
+      }
+    }
+
+    // Let it run for 5 seconds
+    await wait(5000)
+
+    // Unsubscribe all
+    for (const unsub of unsubs) await unsub()
+
+    // Print results
+    console.log('  Camera             | sub0  | sub1  | sub2  | Avg KB')
+    console.log('  -------------------|-------|-------|-------|-------')
+    let allOk = true
+    for (const cam of cameras) {
+      const r = results[cam.id]!
+      const allSizes = [...r[0]!.sizes, ...r[1]!.sizes, ...r[2]!.sizes]
+      const avgKB = allSizes.length > 0 ? +(allSizes.reduce((a, b) => a + b, 0) / allSizes.length / 1024).toFixed(0) : 0
+      const line = `  ${cam.id.padEnd(19)}| ${String(r[0]!.frames).padStart(5)} | ${String(r[1]!.frames).padStart(5)} | ${String(r[2]!.frames).padStart(5)} | ${avgKB}`
+      console.log(line)
+
+      // Every concurrent subscriber must produce frames.
+      for (let sub = 0; sub < subscriberCount; sub++) {
+        if (r[sub]!.frames === 0) allOk = false
+      }
+    }
+
+    // Verify totals
+    const totalFrames = Object.values(results).reduce((sum, cam) =>
+      sum + Object.values(cam).reduce((s, sub) => s + sub.frames, 0), 0)
+    const totalSubs = cameras.length * subscriberCount // 9
+    console.log(`\n  Total: ${totalFrames} frames across ${totalSubs} subscribers`)
+    console.log(`  All subs producing: ${allOk ? 'YES' : 'FAIL'}`)
+
+    // Assertions — every concurrent subscriber on every camera gets frames.
+    for (const cam of cameras) {
+      for (let sub = 0; sub < subscriberCount; sub++) {
+        expect(results[cam.id]![sub]!.frames, `${cam.id} sub${sub} should produce frames`).toBeGreaterThan(0)
+      }
+    }
+
+    // Cleanup
+    for (const id of brokerIds) {
+      const b = manager.getBroker(id)
+      if (b) await b.stop()
+      await manager.destroyBroker(id)
+    }
+  }, 60000)
+
+  it('benchmarks resource usage per stream (CPU, memory, bandwidth)', async () => {
+    console.log('\n--- Resource Benchmark (3 cameras, 5s each) ---')
+
+    const cameras = [
+      { id: 'ingresso_main', url: `rtsp://${FRIGATE}:8554/ingresso_main`, label: 'ingresso main (HD)' },
+      { id: 'ingresso_sub', url: `rtsp://${FRIGATE}:8554/ingresso_sub`, label: 'ingresso sub (SD)' },
+      { id: 'salone_main', url: `rtsp://${FRIGATE}:8554/salone_main`, label: 'salone main (HD)' },
+    ]
+
+    console.log('  Stream              | Enc BW    | Dec FPS | Dec KB/f | CPU %  | RSS MB | Heap MB')
+    console.log('  --------------------|-----------|---------|----------|--------|--------|--------')
+
+    for (const cam of cameras) {
+      const brokerId = `bench-${cam.id}`
+      const broker = await manager.createBroker(brokerId, {
+        type: 'rtsp', url: cam.url, videoCodec: 'h264',
+      })
+      await (broker as any).start({ type: 'rtsp', url: cam.url, videoCodec: 'h264' })
+
+      // Wait for first frame
+      await new Promise<void>((resolve, reject) => {
+        const timeout = setTimeout(() => reject(new Error(`No frame from ${cam.id}`)), 10000)
+        let unsub: (() => Promise<void>) | undefined
+        subscribeDecodedFrames(broker, 10, () => {
+          clearTimeout(timeout)
+          void unsub?.()
+          resolve()
+        }).then((fn) => { unsub = fn }).catch(reject)
+      })
+
+      // Measure baseline
+      const memBefore = process.memoryUsage()
+      const cpuBefore = process.cpuUsage()
+      let encodedBytes = 0
+      let encodedPackets = 0
+      const decodedSizes: number[] = []
+
+      const unsubE = broker.onEncodedData((pkt: EncodedPacket) => {
+        encodedBytes += pkt.data.length
+        encodedPackets++
+      })
+      const unsubD = await subscribeDecodedFrames(broker, 5, (handle: FrameHandle) => {
+        decodedSizes.push(handle.byteLength)
+      })
+
+      // Run for 5 seconds
+      await wait(5000)
+
+      const cpuAfter = process.cpuUsage(cpuBefore)
+      const memAfter = process.memoryUsage()
+
+      unsubE()
+      await unsubD()
+
+      // Calculate metrics
+      const durationS = 5
+      const encBwKBs = (encodedBytes / 1024 / durationS).toFixed(0)
+      const decFps = (decodedSizes.length / durationS).toFixed(1)
+      const avgDecKB = decodedSizes.length > 0
+        ? (decodedSizes.reduce((a, b) => a + b, 0) / decodedSizes.length / 1024).toFixed(0)
+        : '0'
+      // CPU: user + system microseconds → percentage of 5s wall time
+      const cpuTotalUs = cpuAfter.user + cpuAfter.system
+      const cpuPct = (cpuTotalUs / (durationS * 1_000_000) * 100).toFixed(1)
+      const rssDeltaMB = ((memAfter.rss - memBefore.rss) / 1024 / 1024).toFixed(1)
+      const heapMB = (memAfter.heapUsed / 1024 / 1024).toFixed(1)
+
+      const label = cam.label.padEnd(20)
+      console.log(`  ${label}| ${encBwKBs.padStart(5)} KB/s | ${decFps.padStart(7)} | ${avgDecKB.padStart(6)} KB | ${cpuPct.padStart(5)}% | ${rssDeltaMB.padStart(6)} | ${heapMB.padStart(7)}`)
+
+      await broker.stop()
+      await manager.destroyBroker(brokerId)
+
+      // Small pause between cameras for GC
+      await wait(500)
+    }
+
+    // Now test all 3 simultaneously
+    console.log('  --------------------|-----------|---------|----------|--------|--------|--------')
+    console.log('  All 3 concurrent:')
+
+    const memBaseline = process.memoryUsage()
+    const cpuBaseline = process.cpuUsage()
+    const allBrokers: string[] = []
+    const allUnsubs: Array<() => void | Promise<void>> = []
+    let totalEncBytes = 0
+    let totalDecFrames = 0
+
+    for (const cam of cameras) {
+      const brokerId = `bench-all-${cam.id}`
+      allBrokers.push(brokerId)
+      const broker = await manager.createBroker(brokerId, {
+        type: 'rtsp', url: cam.url, videoCodec: 'h264',
+      })
+      await (broker as any).start({ type: 'rtsp', url: cam.url, videoCodec: 'h264' })
+
+      await new Promise<void>((resolve, reject) => {
+        const timeout = setTimeout(() => reject(new Error(`No frame from ${cam.id}`)), 10000)
+        let unsub: (() => Promise<void>) | undefined
+        subscribeDecodedFrames(broker, 10, () => {
+          clearTimeout(timeout)
+          void unsub?.()
+          resolve()
+        }).then((fn) => { unsub = fn }).catch(reject)
+      })
+
+      allUnsubs.push(broker.onEncodedData((pkt: EncodedPacket) => { totalEncBytes += pkt.data.length }))
+      allUnsubs.push(await subscribeDecodedFrames(broker, 5, () => { totalDecFrames++ }))
+    }
+
+    await wait(5000)
+
+    const cpuAll = process.cpuUsage(cpuBaseline)
+    const memAll = process.memoryUsage()
+
+    for (const unsub of allUnsubs) await unsub()
+
+    const allCpuPct = ((cpuAll.user + cpuAll.system) / (5 * 1_000_000) * 100).toFixed(1)
+    const allRssMB = ((memAll.rss - memBaseline.rss) / 1024 / 1024).toFixed(1)
+    const allHeapMB = (memAll.heapUsed / 1024 / 1024).toFixed(1)
+    const allEncKBs = (totalEncBytes / 1024 / 5).toFixed(0)
+
+    console.log(`  Encoded BW: ${allEncKBs} KB/s total`)
+    console.log(`  Decoded: ${totalDecFrames} frames total (${(totalDecFrames / 5).toFixed(1)} fps combined)`)
+    console.log(`  CPU: ${allCpuPct}%`)
+    console.log(`  RSS delta: ${allRssMB} MB`)
+    console.log(`  Heap: ${allHeapMB} MB`)
+
+    for (const id of allBrokers) {
+      const b = manager.getBroker(id)
+      if (b) await b.stop()
+      await manager.destroyBroker(id)
+    }
+
+    // Basic sanity — we should be able to handle 3 streams
+    expect(totalDecFrames).toBeGreaterThan(10)
+  }, 90000)
+})