@vpalmisano/webrtcperf 4.4.9 → 4.4.11

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vpalmisano/webrtcperf",
-  "version": "4.4.9",
+  "version": "4.4.11",
   "repository": {
     "type": "git",
     "url": "https://github.com/vpalmisano/webrtcperf.git"
@@ -51,11 +51,11 @@
   },
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@google/genai": "^1.17.0",
-    "@puppeteer/browsers": "^2.10.8",
+    "@google/genai": "^1.20.0",
+    "@puppeteer/browsers": "^2.10.10",
     "@vpalmisano/throttler": "0.0.14",
-    "@vpalmisano/webrtcperf-js": "^1.1.13",
-    "axios": "^1.11.0",
+    "@vpalmisano/webrtcperf-js": "^1.1.14",
+    "axios": "^1.12.2",
     "chalk-template": "^1.1.2",
     "change-case": "^4.1.2",
     "compression": "^1.8.1",
@@ -67,23 +67,24 @@
     "express-basic-auth": "^1.2.1",
     "fast-stats": "^0.0.7",
     "form-data": "^4.0.4",
+    "googleapis": "^160.0.0",
     "ipaddr.js": "^2.2.0",
     "json5": "^2.2.3",
     "lorem-ipsum": "^2.0.8",
-    "marked": "^16.2.1",
+    "marked": "^16.3.0",
     "marked-terminal": "^7.3.0",
     "node-cache": "^5.1.2",
     "node-os-utils": "^1.3.7",
     "pidtree": "^0.6.0",
     "pidusage": "^4.0.1",
     "prom-client": "^15.1.3",
-    "puppeteer": "^24.19.0",
-    "puppeteer-core": "^24.19.0",
+    "puppeteer": "^24.22.0",
+    "puppeteer-core": "^24.22.0",
     "puppeteer-extra": "^3.3.6",
     "puppeteer-extra-plugin-stealth": "^2.11.2",
     "puppeteer-intercept-and-modify-requests": "^1.3.1",
     "sprintf-js": "^1.1.3",
-    "tar-fs": "^3.1.0",
+    "tar-fs": "^3.1.1",
     "toml": "^3.0.0",
     "ws": "^8.18.3",
     "yaml": "^2.8.1"
@@ -94,18 +95,18 @@
     "@types/compression": "^1.8.1",
     "@types/convict": "^6.1.6",
     "@types/convict-format-with-validator": "^6.0.5",
-    "@types/dockerode": "^3.3.43",
+    "@types/dockerode": "^3.3.44",
     "@types/fast-stats": "^0.0.35",
     "@types/marked-terminal": "^6.1.1",
-    "@types/node": "^20.19.13",
+    "@types/node": "^20.19.17",
     "@types/node-os-utils": "^1.3.4",
     "@types/pidusage": "^2.0.5",
     "@types/sdp-transform": "^2.15.0",
     "@types/sprintf-js": "^1.1.4",
     "@types/tar-fs": "^2.0.4",
     "@types/ws": "^8.18.1",
-    "@typescript-eslint/eslint-plugin": "^8.43.0",
-    "@typescript-eslint/parser": "^8.43.0",
+    "@typescript-eslint/eslint-plugin": "^8.44.0",
+    "@typescript-eslint/parser": "^8.44.0",
     "@vpalmisano/typedoc-cookie-consent": "^0.0.4",
     "@vpalmisano/typedoc-plugin-ga": "^1.0.6",
     "eslint": "^9.35.0",
@@ -119,9 +120,9 @@
     "prettier": "^3.6.2",
     "terser-webpack-plugin": "^5.3.14",
     "ts-loader": "^9.5.4",
-    "typedoc": "^0.28.12",
+    "typedoc": "^0.28.13",
     "typescript": "^5.9.2",
-    "typescript-eslint": "^8.43.0",
+    "typescript-eslint": "^8.44.0",
     "webpack": "^5.101.3",
     "webpack-cli": "^6.0.1",
     "webpack-node-externals": "^3.0.0",

package/src/docker.ts

@@ -3,6 +3,7 @@ import { logger, resolvePackagePath } from './utils'
 import { loadConfig } from './config'
 import { runShellCommand } from '@vpalmisano/throttler'
 import os from 'os'
+import fs from 'fs'
 
 const log = logger('webrtcperf:docker')
 
@@ -10,12 +11,22 @@ export async function runWithDocker(argv: string[]) {
   const docker = new Docker()
   const configPath = argv[0]
   if (!configPath) throw new Error('No configuration file specified')
-  const config = (await loadConfig(configPath))[0]
+  const configs = await loadConfig(configPath)
+  if (!configs.length) throw new Error('Failed to load configuration file')
 
   const startTimestamp = Date.now()
   const dataDir = process.cwd()
+  const tmpDir = os.tmpdir()
 
-  const binds: string[] = ['/dev/shm:/dev/shm', `${dataDir}:/data`, '/tmp/webrtcperf-cache:/root/.webrtcperf']
+  const jsonConfigPath = `${tmpDir}/webrtcperf-config-${startTimestamp}.json`
+  await fs.promises.writeFile(jsonConfigPath, JSON.stringify(configs), 'utf-8')
+
+  const binds: string[] = [
+    '/dev/shm:/dev/shm',
+    `${dataDir}:/data`,
+    `${tmpDir}/webrtcperf-cache:/root/.webrtcperf`,
+    `${jsonConfigPath}:/tmp/config.json:ro`,
+  ]
 
   if (process.env.DEBUG_SRC) {
     binds.push(`${resolvePackagePath('app.min.js')}:/app/app.min.js:ro`)
@@ -23,15 +34,15 @@ export async function runWithDocker(argv: string[]) {
 
   const portBindings: Docker.PortMap = {}
   const exposedPorts: { [portAndProtocol: string]: object } = {}
-  if (config.debuggingPort) {
-    for (let i = 0; i < config.sessions; i++) {
-      const port = `${config.debuggingPort + i}/tcp`
-      portBindings[port] = [{ HostPort: `${config.debuggingPort + i}` }]
+  if (configs[0].debuggingPort) {
+    for (let i = 0; i < configs[0].sessions; i++) {
+      const port = `${configs[0].debuggingPort + i}/tcp`
+      portBindings[port] = [{ HostPort: `${configs[0].debuggingPort + i}` }]
       exposedPorts[port] = {}
     }
   }
 
-  if (config.throttleConfig && os.platform() === 'linux') {
+  if (configs[0].throttleConfig && os.platform() === 'linux') {
     await runShellCommand('sudo modprobe ifb numifbs=1')
   }
 
@@ -45,7 +56,7 @@ export async function runWithDocker(argv: string[]) {
     `START_TIMESTAMP=${startTimestamp}`,
   ]
 
-  if (config.prometheusPushgateway.startsWith('http://localhost')) {
+  if (configs[0].prometheusPushgateway.startsWith('http://localhost')) {
     env.push('PROMETHEUS_PUSHGATEWAY=http://pushgateway:9091')
   }
 
@@ -53,12 +64,14 @@ export async function runWithDocker(argv: string[]) {
     Image: 'ghcr.io/vpalmisano/webrtcperf:devel',
     name: 'webrtcperf',
     WorkingDir: '/data',
-    Cmd: argv,
+    Cmd: ['/tmp/config.json'],
     HostConfig: {
       Binds: binds,
       PortBindings: portBindings,
-      CapAdd: config.throttleConfig && os.platform() === 'linux' ? ['NET_ADMIN'] : [],
-      NetworkMode: config.prometheusPushgateway.startsWith('http://localhost') ? 'prometheus-stack_default' : 'bridge',
+      CapAdd: configs[0].throttleConfig && os.platform() === 'linux' ? ['NET_ADMIN'] : [],
+      NetworkMode: configs[0].prometheusPushgateway.startsWith('http://localhost')
+        ? 'prometheus-stack_default'
+        : 'bridge',
       ExtraHosts: process.env.EXTRA_HOSTS ? process.env.EXTRA_HOSTS.split(',').map(h => h.trim()) : [],
     },
     Env: env,
@@ -110,4 +123,6 @@ export async function runWithDocker(argv: string[]) {
     log.error('Docker operation failed:', error)
     throw error
   }
+
+  await fs.promises.unlink(jsonConfigPath)
 }

package/src/index.ts

@@ -1,5 +1,6 @@
 export * from './app'
 export * from './config'
+export * from './docker'
 export * from './media'
 export * from './rtcstats'
 export * from './server'
@@ -7,3 +8,4 @@ export * from './session'
 export * from './stats'
 export * from './utils'
 export * from './vmaf'
+export * from './scenarios'

package/src/scenarios.ts

@@ -0,0 +1,262 @@
+import fs from 'fs'
+import path from 'path'
+import { FastStats } from './stats'
+import { ThrottleConfig, ThrottleRule } from '@vpalmisano/throttler'
+import { Config } from './config'
+import { Auth, google } from 'googleapis'
+import { logger } from './utils'
+
+const log = logger('webrtcperf:scenarios')
+
+/**
+ * It parses a CSV stats file and returns an array of objects representing each row.
+ * @param filePath The path to the CSV stats file.
+ * @returns An array of objects where each object represents a row in the CSV file with keys as column headers.
+ */
+export async function parseStatsFile(filePath: string) {
+  log.debug(`parseStatsFile: ${filePath}`)
+  const fileData = await fs.promises.readFile(filePath, 'utf-8')
+  const lines = fileData.split('\n')
+  const headers = lines[0].split(',')
+  const data = lines.slice(1).map(line =>
+    line.split(',').reduce(
+      (acc, value, index) => {
+        if (value !== '') {
+          acc[headers[index]] = isNaN(Number(value)) ? value : Number(value)
+        }
+        return acc
+      },
+      {} as Record<string, string | number>,
+    ),
+  )
+  return data
+}
+
+export type StatsSummary = {
+  timestamp: number
+  id: string
+  scenario: string
+  videoRecvBitratePerPixel: FastStats
+  videoRecvFps: FastStats
+  videoSentFps: FastStats
+}
+
+/**
+ * It aggregates the stats summary from multiple test runs in a directory.
+ * @param options.dirPath Directory path containing test run subdirectories. Default is 'logs'.
+ * @param options.senderParticipantName Participant name of the sender. Default is 'Participant-000001'.
+ * @param options.receiverParticipantName Participant name of the receiver. Default is 'Participant-000000'.
+ * @param options.nameParser Function to parse test directory names. Default splits by '_' and extracts id and scenario.
+ * @returns Array of aggregated stats including timestamp, id, scenario, videoRecvBitratePerPixel, videoRecvFps, and videoSentFps.
+ */
+export async function aggregateStatsSummary({
+  dirPath = 'logs',
+  senderParticipantName = 'Participant-000001',
+  receiverParticipantName = 'Participant-000000',
+  nameParser = (name: string) => {
+    const [_, id, scenario] = name.split('_')
+    return { id, scenario }
+  },
+}) {
+  log.debug(`aggregateStatsSummary: ${dirPath}`)
+  const stats: StatsSummary[] = []
+  const results = await fs.promises.readdir(dirPath)
+  for (const test of results) {
+    const filePath = path.join(dirPath, test, 'detailed-stats-summary.csv')
+    if (!fs.existsSync(filePath)) continue
+    const timestamp = fs.statSync(path.join(dirPath, test)).ctime.getTime()
+    const data = await parseStatsFile(filePath)
+    const { id, scenario } = nameParser(test)
+
+    const aggregated = {
+      timestamp,
+      id,
+      scenario,
+      videoRecvBitratePerPixel: new FastStats(),
+      videoRecvFps: new FastStats(),
+      videoSentFps: new FastStats(),
+    }
+    data.forEach(v => {
+      const { participantName, trackId } = v as { participantName: string; trackId: string }
+      const metrics = v as Record<string, number>
+      if (participantName === receiverParticipantName) {
+        if (trackId?.endsWith('-v') && metrics.videoRecvFrames > 0) {
+          const videoRecvBitratePerPixel =
+            metrics.videoRecvBitrates / (metrics.videoRecvWidth * metrics.videoRecvHeight)
+          if (!isNaN(videoRecvBitratePerPixel)) aggregated.videoRecvBitratePerPixel.push(videoRecvBitratePerPixel)
+          if (!isNaN(metrics.videoRecvFps)) aggregated.videoRecvFps.push(metrics.videoRecvFps)
+        }
+      } else if (participantName === senderParticipantName) {
+        if (trackId?.endsWith('-v') && metrics.videoSentFrames > 0) {
+          if (!isNaN(metrics.videoSentFps)) aggregated.videoSentFps.push(metrics.videoSentFps)
+        }
+      }
+    })
+    stats.push(aggregated)
+  }
+  return stats.sort((a, b) => a.timestamp - b.timestamp)
+}
+
+/**
+ * It uploads the aggregated stats to a Google Sheet.
+ * A valid Google service account credentials file must be specified
+ * in the `GOOGLE_CREDENTIALS_PATH` environment variable.
+ * @param stats The aggregated stats to upload.
+ * @param spreadsheetId The ID of the Google Spreadsheet.
+ * @param table The name of the table (sheet) within the spreadsheet. Default is 'data'.
+ */
+export async function uploadStatsToGoogleSheet(stats: StatsSummary[], spreadsheetId: string, table = 'data') {
+  log.debug(`uploadResultsToGoogleSheet spreadsheetId: ${spreadsheetId} table: ${table}`)
+  if (!process.env.GOOGLE_CREDENTIALS_PATH) throw new Error('GOOGLE_CREDENTIALS_PATH environment variable is not set')
+  if (!fs.existsSync(process.env.GOOGLE_CREDENTIALS_PATH))
+    throw new Error(`Google credentials file not found: ${process.env.GOOGLE_CREDENTIALS_PATH}`)
+  if (!stats.length) return
+  const auth = new Auth.GoogleAuth({
+    keyFile: process.env.GOOGLE_CREDENTIALS_PATH,
+    scopes: ['https://www.googleapis.com/auth/spreadsheets'],
+  })
+  const sheets = google.sheets({ version: 'v4', auth })
+  // Update headers.
+  const headers = ['datetime', 'id', 'scenario', 'videoRecvBitratePerPixel', 'videoRecvFps']
+  await sheets.spreadsheets.values.update({
+    spreadsheetId,
+    range: `${table}!A1:E1`,
+    valueInputOption: 'USER_ENTERED',
+    requestBody: { majorDimension: 'ROWS', values: [headers] },
+  })
+  // Append values.
+  const values = [] as string[][]
+  stats.forEach(s => {
+    const { timestamp, id, scenario, videoRecvBitratePerPixel, videoRecvFps } = s
+    if (!videoRecvBitratePerPixel.length) return
+    const datetime = new Date(timestamp).toLocaleString('en-US', {
+      timeZone: 'UTC',
+      hourCycle: 'h23',
+    })
+    values.push([
+      datetime,
+      id,
+      formatThrottleRule(parseThrottleRule(scenario), true),
+      videoRecvBitratePerPixel.percentile(95).toFixed(3),
+      videoRecvFps.percentile(95).toFixed(3),
+    ])
+  })
+  if (values.length) {
+    await sheets.spreadsheets.values.append({
+      spreadsheetId,
+      range: `${table}!A:E`,
+      valueInputOption: 'USER_ENTERED',
+      insertDataOption: 'INSERT_ROWS',
+      requestBody: { majorDimension: 'ROWS', values },
+    })
+  }
+}
+
+export type ThrottleDirection = 'up' | 'down' | 'bidi'
+
+function formatBitrate(bitrate: number | undefined, prefix = ' ') {
+  if (bitrate === undefined) return ''
+  let suffix = 'Kbps'
+  if (bitrate >= 10000) {
+    bitrate /= 1000
+    suffix = 'Mbps'
+  }
+  return `${prefix}${bitrate.toFixed(0)}${suffix}`.padStart(8, ' ')
+}
+
+function formatLoss(loss: number | undefined, prefix = ' ') {
+  return loss !== undefined ? `${prefix}${loss.toFixed(0).padStart(2, ' ')}%` : ''
+}
+
+function formatDelay(delay: number | undefined, prefix = ' ') {
+  return delay !== undefined ? `${prefix}${delay.toFixed(0).padStart(3, ' ')}ms` : ''
+}
+
+export function formatThrottleRule(throttleRule: ThrottleRule & { direction: ThrottleDirection }, human = false) {
+  const { rate, loss, delay, direction } = throttleRule
+  return human
+    ? `${direction.padEnd(4, ' ')}${formatBitrate(rate)}${formatLoss(loss)}${formatDelay(delay)}`
+    : `${direction}-r${rate}-l${loss}-d${delay}`
+}
+
+export function parseThrottleRule(throttleDesc: string) {
+  const match = throttleDesc.match(/(up|down|bidi)-r(\d+)-l([\d.]+)-d(\d+)/)
+  if (!match) throw new Error(`Invalid throttle description: ${throttleDesc}`)
+  const direction = match[1] as ThrottleDirection
+  const rate = parseInt(match[2])
+  const loss = parseInt(match[3])
+  const delay = parseInt(match[4])
+  return { direction, rate, loss, delay }
+}
+
+/**
+ * It generates a test configuration with a scenario including 2 participants.
+ * The first participant sends video and the second receives it.
+ * Both participants send and receive audio.
+ * The network conditions are applied according to the specified direction to the sender (`up`),
+ * the receiver (`down`) or both (`bidi`).
+ * The test is repeated the specified number of times.
+ * The output is an array of partial configuration objects that can be used to run the tests
+ * with the main application, after merging it with a configuration that includes
+ * the destination url (mandatory) and other optional parameters.
+ * @param id The unique identifier for the test scenario.
+ * @param options.rate The target bandwidth in kbps.
+ * @param options.loss The packet loss percentage.
+ * @param options.delay The network delay in milliseconds.
+ * @param options.direction The direction of the network throttling: 'up', 'down', or 'bidi'.
+ * @param repeat The number of times to repeat the test scenario. Default is 1.
+ * @returns An array of partial configuration objects for each test scenario.
+ */
+export async function twoParticipantsWithRateLossDelay(
+  id: string,
+  { rate, loss, delay, direction }: { rate: number; loss: number; delay: number; direction: ThrottleDirection },
+  repeat: 1,
+) {
+  const throttle: ThrottleConfig = {}
+  const queue = 25
+  if (direction === 'down' || direction === 'bidi') {
+    throttle.down = [
+      { rate: 20000, loss: 0, delay: 0, queue },
+      { rate, loss, delay, queue, at: 30 },
+    ]
+  }
+  if (direction === 'up' || direction === 'bidi') {
+    throttle.up = [
+      { rate: 20000, loss: 0, delay, queue },
+      { rate, loss, delay, queue, at: 30 },
+    ]
+  }
+  const throttleDesc = formatThrottleRule({ rate, loss, delay, direction })
+  const now = Date.now()
+  const ret: Partial<Config>[] = []
+  for (let i = 0; i < repeat; i++) {
+    const basePath = `logs/${now}-${i + 1}_${id}_${throttleDesc}`
+    const sessions = direction === 'bidi' ? '0-1' : direction === 'down' ? '0' : '1'
+    ret.push({
+      sessions: 2,
+      runDuration: 60 * 3,
+      debuggingPort: 9000,
+      prometheusPushgateway: 'http://localhost:9091',
+      prometheusPushgatewayJobName: id,
+      statsPath: `${basePath}/stats.csv`,
+      detailedStatsPath: `${basePath}/detailed-stats.csv`,
+      showPageLog: false,
+      showStats: false,
+      statsInterval: 5,
+      scriptParams: JSON.stringify({
+        enableMic: '0-1',
+        enableCam: '1',
+      }),
+      throttleConfig: JSON.stringify([
+        {
+          sessions,
+          protocol: 'udp',
+          skipSourcePorts: '53,80,443',
+          skipDestinationPorts: '53,80,443',
+          ...throttle,
+        },
+      ]),
+    })
+  }
+  return ret
+}

package/src/utils.ts

@@ -23,7 +23,6 @@ import pidusage from 'pidusage'
 import puppeteer, { ImageFormat, Page } from 'puppeteer-core'
 
 import { Session } from './session'
-import { FastStats } from './stats'
 
 // eslint-disable-next-line
 const ps = require('pidusage/lib/ps')
@@ -1256,75 +1255,3 @@ export async function getDockerLogsPath(): Promise<string> {
   }
   return logPath
 }
-
-export async function parseStatsFile(filePath: string) {
-  const fileData = await fs.promises.readFile(filePath, 'utf-8')
-  const lines = fileData.split('\n')
-  const headers = lines[0].split(',')
-  const data = lines.slice(1).map(line =>
-    line.split(',').reduce(
-      (acc, value, index) => {
-        if (value !== '') {
-          acc[headers[index]] = isNaN(Number(value)) ? value : Number(value)
-        }
-        return acc
-      },
-      {} as Record<string, string | number>,
-    ),
-  )
-  return data
-}
-
-export async function aggregateStatsSummary({
-  dirPath = 'logs',
-  senderParticipantName = 'Participant-000001',
-  receiverParticipantName = 'Participant-000000',
-  nameParser = (name: string) => {
-    const [destination, scenario] = name.split('_')
-    return { destination, scenario }
-  },
-}) {
-  const stats = [] as {
-    timestamp: number
-    destination: string
-    scenario: string
-    videoRecvBitratePerPixel: FastStats
-    videoRecvFps: FastStats
-    videoSentFps: FastStats
-  }[]
-  const results = await fs.promises.readdir(dirPath)
-  for (const test of results) {
-    const filePath = path.join(dirPath, test, 'detailed-stats-summary.csv')
-    if (!fs.existsSync(filePath)) continue
-    const timestamp = fs.statSync(path.join(dirPath, test)).ctime.getTime()
-    const data = await parseStatsFile(filePath)
-    const { destination, scenario } = nameParser(test)
-
-    const aggregated = {
-      timestamp,
-      destination,
-      scenario,
-      videoRecvBitratePerPixel: new FastStats(),
-      videoRecvFps: new FastStats(),
-      videoSentFps: new FastStats(),
-    }
-    data.forEach(v => {
-      const { participantName, trackId } = v as { participantName: string; trackId: string }
-      const metrics = v as Record<string, number>
-      if (participantName === receiverParticipantName) {
-        if (trackId?.endsWith('-v') && metrics.videoRecvFrames > 0) {
-          const videoRecvBitratePerPixel =
-            metrics.videoRecvBitrates / (metrics.videoRecvWidth * metrics.videoRecvHeight)
-          if (!isNaN(videoRecvBitratePerPixel)) aggregated.videoRecvBitratePerPixel.push(videoRecvBitratePerPixel)
-          if (!isNaN(metrics.videoRecvFps)) aggregated.videoRecvFps.push(metrics.videoRecvFps)
-        }
-      } else if (participantName === senderParticipantName) {
-        if (trackId?.endsWith('-v') && metrics.videoSentFrames > 0) {
-          if (!isNaN(metrics.videoSentFps)) aggregated.videoSentFps.push(metrics.videoSentFps)
-        }
-      }
-    })
-    stats.push(aggregated)
-  }
-  return stats.sort((a, b) => a.timestamp - b.timestamp)
-}