@vpalmisano/webrtcperf 4.2.2 → 4.4.1

package/src/app.ts

@@ -23,6 +23,8 @@ import { calculateVisqolScore } from './visqol'
 import { calculateVmafScore, convertToIvf, prepareVideo } from './vmaf'
 import path from 'path'
 import { markedTerminal } from 'marked-terminal'
+import { EventEmitter } from 'events'
+import { runWithDocker } from './docker'
 
 // eslint-disable-next-line @typescript-eslint/no-require-imports
 const { marked } = require('marked')
@@ -56,141 +58,167 @@ Default value: \`${value.default}\`
   }
 }
 
-async function postTest(config: Config): Promise<void> {
-  // vmaf score.
-  if (config.vmafPath) {
-    console.log('Calculating VMAF score...')
-    try {
-      await calculateVmafScore(config)
-    } catch (err: unknown) {
-      log.error(`vmaf score error: ${(err as Error).stack}`)
+export class Application extends EventEmitter {
+  readonly config: Config
+  readonly stats: Stats
+  readonly server?: Server
+  private mediaPaths: MediaPath[] = []
+
+  constructor(config: Config) {
+    super()
+    if (!config.startTimestamp) {
+      config.startTimestamp = Date.now()
+    }
+    this.config = config
+    this.stats = new Stats(config)
+    if (config.serverPort) {
+      this.server = new Server(config, this.stats)
     }
   }
 
-  // visqol score
-  if (config.visqolPath) {
-    console.log('Calculating Visqol score...')
-    try {
-      await calculateVisqolScore(config)
-    } catch (err: unknown) {
-      log.error(`visqol score error: ${(err as Error).stack}`)
+  async start() {
+    log.debug(`start (runDuration: ${this.config.runDuration})`)
+    await this.stats.start()
+    if (this.server) {
+      await this.server.start()
     }
-  }
-}
+    const config = this.config
 
-export async function setupApplication(config: Config): Promise<{ stats: Stats; stop: () => Promise<void> }> {
-  if (!config.startTimestamp) {
-    config.startTimestamp = Date.now()
-  }
+    // Handle vmaf commands.
+    if (config.vmafPrepareVideo) {
+      await prepareVideo(config, true)
+    }
+    if (config.vmafProcessVideo) {
+      await convertToIvf(
+        config.vmafProcessVideo,
+        config.vmafVideoCrop,
+        config.vmafKeepSourceFiles,
+        config.vmafSkipDuplicated,
+      )
+    }
 
-  // Stats.
-  const stats = new Stats(config)
-  await stats.start()
+    // Handle sessions.
+    if (config.sessions > 0) {
+      // Prepare fake video and audio.
+      if (config.videoPath && !this.mediaPaths.length) {
+        for (const videoPath of config.videoPath.split(',')) {
+          const ret = await prepareFakeMedia({ ...config, videoPath })
+          this.mediaPaths.push(ret)
+        }
+      }
 
-  // Control server.
-  let server: Server | undefined
-  if (config.serverPort) {
-    server = new Server(config, stats)
-    await server.start()
-  }
+      // Network throttle.
+      if (config.throttleConfig) {
+        await startThrottle(config.throttleConfig)
+      }
 
-  // If sessions are set, prepare fake video/audio and start sessions.
-  if (config.sessions > 0) {
-    // Prepare fake video and audio.
-    const mediaPaths: MediaPath[] = []
-    if (config.videoPath) {
-      for (const videoPath of config.videoPath.split(',')) {
-        const ret = await prepareFakeMedia({ ...config, videoPath })
-        mediaPaths.push(ret)
+      // Download browser if necessary.
+      if (!config.chromiumUrl && !config.chromiumPath) {
+        await checkChromeExecutable()
       }
-    }
 
-    // Network throttle.
-    if (config.throttleConfig) {
-      await startThrottle(config.throttleConfig)
+      // Start the local sessions.
+      if (config.randomAudioPeriod) {
+        startRandomActivateAudio(
+          this.stats.sessions,
+          config.randomAudioPeriod,
+          config.randomAudioProbability,
+          config.randomAudioRange,
+        )
+      }
+      const spawnPeriod = 1000 / config.spawnRate
+      log.debug(`Starting ${config.sessions} sessions (spawnPeriod: ${spawnPeriod}ms)`)
+      const startTime = Date.now()
+      for (let i = 0; i < config.sessions; i += 1) {
+        const id = this.stats.consumeSessionId(config.tabsPerSession)
+        await this.startSession(id, spawnPeriod)
+        // If not the last session, sleep.
+        if (i < config.sessions - 1) {
+          await sleep(spawnPeriod)
+        }
+      }
+      const elapsed = Math.round((Date.now() - startTime) / 1000)
+      const spawnRate = (config.sessions * config.tabsPerSession) / elapsed
+      log.debug(`${config.sessions * config.tabsPerSession} pages started in ${elapsed}s (${spawnRate.toFixed(2)}/s)`)
     }
 
-    // Download browser if necessary.
-    if (!config.chromiumUrl && !config.chromiumPath) {
-      await checkChromeExecutable()
+    if (config.runDuration || config.vmafPath || config.visqolPath) {
+      setTimeout(() => this.stop(), config.runDuration * 1000)
     }
+  }
 
-    // Start session function.
-    const startLocalSession = async (id: number, spawnPeriod: number): Promise<void> => {
-      const throttleIndex = getSessionThrottleIndex(id)
-      const mediaPath = mediaPaths.length ? mediaPaths[id % mediaPaths.length] : undefined
-      const session = new Session({
-        ...config,
-        mediaPath,
-        spawnPeriod,
-        id,
-        throttleIndex,
-      })
-      session.once('stop', () => {
-        console.warn(`Session ${id} stopped, reloading...`)
-        setTimeout(startLocalSession, spawnPeriod, id)
-      })
-      stats.addSession(session)
-      await session.start()
-    }
+  private async startSession(id: number, spawnPeriod: number) {
+    log.debug(`startSession ${id}`)
+    const throttleIndex = getSessionThrottleIndex(id)
+    const mediaPath = this.mediaPaths.length ? this.mediaPaths[id % this.mediaPaths.length] : undefined
+    const session = new Session({
+      ...this.config,
+      mediaPath,
+      spawnPeriod,
+      id,
+      throttleIndex,
+    })
+    session.once('stop', () => {
+      console.warn(`Session ${id} stopped, reloading...`)
+      setTimeout(() => this.startSession(id, spawnPeriod), spawnPeriod)
+    })
+    this.stats.addSession(session)
+    await session.start()
+  }
 
-    // Start the local sessions.
-    if (config.randomAudioPeriod) {
-      startRandomActivateAudio(
-        stats.sessions,
-        config.randomAudioPeriod,
-        config.randomAudioProbability,
-        config.randomAudioRange,
-      )
+  private async postTest() {
+    log.debug('postTest')
+
+    // vmaf score.
+    if (this.config.vmafPath) {
+      console.log('Calculating VMAF score...')
+      try {
+        await calculateVmafScore(this.config)
+      } catch (err: unknown) {
+        log.error(`vmaf score error: ${(err as Error).stack}`)
+      }
     }
-    const spawnPeriod = 1000 / config.spawnRate
-    log.debug(`Starting ${config.sessions} sessions (spawnPeriod: ${spawnPeriod}ms)`)
-    const startTime = Date.now()
-    for (let i = 0; i < config.sessions; i += 1) {
-      const id = stats.consumeSessionId(config.tabsPerSession)
-      await startLocalSession(id, spawnPeriod)
-      // If not the last session, sleep
-      if (i < config.sessions - 1) {
-        await sleep(spawnPeriod)
+
+    // visqol score
+    if (this.config.visqolPath) {
+      console.log('Calculating Visqol score...')
+      try {
+        await calculateVisqolScore(this.config)
+      } catch (err: unknown) {
+        log.error(`visqol score error: ${(err as Error).stack}`)
       }
     }
-    const elapsed = Math.round((Date.now() - startTime) / 1000)
-    const spawnRate = (config.sessions * config.tabsPerSession) / elapsed
-    log.debug(`${config.sessions * config.tabsPerSession} pages started in ${elapsed}s (${spawnRate.toFixed(2)}/s)`)
   }
 
-  return {
-    stats,
-    stop: async (): Promise<void> => {
-      log.debug('Stopping')
+  async stop(canceled = false) {
+    log.debug(`stop (canceled: ${canceled})`)
 
-      stopRandomActivateAudio()
+    stopRandomActivateAudio()
 
-      await stats.stop()
+    await this.stats.stop()
 
-      if (config.throttleConfig) {
-        await stopThrottle()
-      }
+    if (this.config.throttleConfig) {
+      await stopThrottle()
+    }
 
-      stopTimers()
+    stopTimers()
 
-      await postTest(config)
+    await this.postTest()
 
-      // Copy docker logs to data directory.
-      if (config.pageLogPath) {
-        try {
-          const logPath = await getDockerLogsPath()
-          const dataDir = path.dirname(config.pageLogPath)
-          await fs.promises.cp(logPath, path.resolve(dataDir, 'docker.log'))
-        } catch (err: unknown) {
-          log.debug(`docker logs not found: ${(err as Error).message}`)
-        }
+    // Copy docker logs to data directory.
+    if (this.config.pageLogPath) {
+      try {
+        const logPath = await getDockerLogsPath()
+        const dataDir = path.dirname(this.config.pageLogPath)
+        await fs.promises.cp(logPath, path.resolve(dataDir, 'docker.log'))
+      } catch (err: unknown) {
+        log.debug(`docker logs not found: ${(err as Error).message}`)
       }
+    }
 
-      server?.stop()
+    this.server?.stop()
 
-      log.debug('Stopped')
-    },
+    this.emit('stop', canceled)
   }
 }
 
@@ -200,52 +228,58 @@ export async function setupApplication(config: Config): Promise<{ stats: Stats;
 async function main(): Promise<void> {
   showHelpOrVersion()
 
-  let config: Config
+  const argv = process.argv.slice(2)
 
-  if (process.argv.slice(2).includes('--prompt')) {
-    const params = await loadConfigFromPrompt(
-      process.argv
-        .slice(2)
-        .filter(s => !['--prompt', '--dry-run'].includes(s))
-        .join(' '),
-    )
+  // Handle docker run.
+  if (argv.includes('--docker')) {
+    try {
+      await runWithDocker(argv)
+    } catch (err: unknown) {
+      log.error(`runWithDocker error: ${(err as Error).stack}`)
+      process.exit(1)
+    }
+    process.exit(0)
+  }
+
+  let configs: Config[]
+
+  // Handle prompt.
+  if (argv.includes('--prompt')) {
+    const params = await loadConfigFromPrompt(argv.filter(s => !['--prompt', '--dry-run'].includes(s)).join(' '))
     if (process.argv.slice(2).includes('--dry-run')) {
       console.log(json5.stringify(params, null, 2))
       process.exit(0)
     }
-    config = await loadConfig(undefined, params)
+    configs = await loadConfig(undefined, params)
   } else {
-    config = await loadConfig(process.argv[2])
+    configs = await loadConfig(process.argv[2])
   }
 
-  if (config.vmafPrepareVideo) {
-    await prepareVideo(config, true)
-    process.exit(0)
-  }
+  if (!configs.length) throw new Error('No configuration found')
 
-  if (config.vmafProcessVideo) {
-    await convertToIvf(
-      config.vmafProcessVideo,
-      config.vmafVideoCrop,
-      config.vmafKeepSourceFiles,
-      config.vmafSkipDuplicated,
-    )
-    process.exit(0)
-  }
+  let application: Application
+  const runNext = () => {
+    const config = configs.splice(0, 1)[0]
 
-  const { stop: stopApplication } = await setupApplication(config)
+    application = new Application(config)
+    application.once('stop', canceled => {
+      if (!canceled && configs.length) {
+        log.info(`Application stopped, running next (${configs.length} left)...`)
+        runNext()
+      } else {
+        process.exit(0)
+      }
+    })
+    return application.start()
+  }
 
-  const stop = async (): Promise<void> => {
+  const stop = async () => {
     console.log('Exiting...')
-
-    await stopApplication()
-
-    process.exit(0)
+    await application.stop(true)
   }
   registerExitHandler(() => stop())
 
-  // Stop after a configured duration.
-  setTimeout(stop, config.runDuration * 1000)
+  await runNext()
 
   // Command line interface.
   if (process.stdin && process.stdin.setRawMode) {

package/src/config.ts

@@ -6,6 +6,7 @@ import path, { join } from 'path'
 import json5 from 'json5'
 import yaml from 'yaml'
 import toml from 'toml'
+import fs from 'fs'
 
 // eslint-disable-next-line @typescript-eslint/no-require-imports
 const puppeteer = require('puppeteer-core')
@@ -56,8 +57,7 @@ convict.addParser([
   { extension: 'toml', parse: toml.parse },
 ])
 
-// config schema
-const configSchema = convict({
+const configSchema = {
   url: {
     doc: `The page url to load.`,
     format: String,
@@ -859,7 +859,7 @@ E.g. \`{ w: "iw-10", h: "ih-5", x: "10", y: '5' }\``,
     env: 'VISQOL_KEEP_SOURCE_FILES',
     arg: 'visqol-keep-source-files',
   },
-})
+}
 
 type ConfigDocs = Record<string, { doc: string; format: string; default: string }>
 
@@ -897,10 +897,10 @@ function formatDocs(
  * It returns the formatted configuration docs.
  */
 export function getConfigDocs(): ConfigDocs {
-  return formatDocs({}, null, configSchema.getSchema())
+  return formatDocs({}, null, convict(configSchema).getSchema())
 }
 
-const _schemaProperties = configSchema.getProperties()
+const _schemaProperties = convict(configSchema).getProperties()
 
 /** [[include:config.md]] */
 export type Config = typeof _schemaProperties
@@ -909,7 +909,8 @@ export type Config = typeof _schemaProperties
  * Loads the config object.
  */
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
-export async function loadConfig(filePath?: string, values?: any): Promise<Config> {
+export async function loadConfig(filePath?: string, values?: any): Promise<Config[]> {
+  const configs: Config[] = []
   if (filePath) {
     if (filePath.startsWith('http')) {
       log.debug(`Loading config from url: ${filePath}`)
@@ -917,41 +918,45 @@ export async function loadConfig(filePath?: string, values?: any): Promise<Confi
       if (!res?.data) {
         throw new Error(`Failed to download configuration from: ${filePath}`)
       }
-      const values =
+      values =
         res.contentType === 'application/x-yaml'
           ? yaml.parse(res.data)
           : res.contentType === 'application/toml'
             ? toml.parse(res.data)
             : json5.parse(res.data)
-      configSchema.load(values)
     } else if (existsSync(filePath)) {
       log.debug(`Loading config from local file: ${filePath}`)
       if (filePath.endsWith('.js') || filePath.endsWith('.mjs')) {
         const module = await import(/* webpackIgnore: true */ path.resolve(filePath))
-        configSchema.load(await module.default())
+        values = await module.default()
       } else {
-        configSchema.loadFile(filePath)
+        const data = String(await fs.promises.readFile(filePath))
+        values =
+          filePath.endsWith('.yml') || filePath.endsWith('.yaml')
+            ? yaml.parse(data)
+            : filePath.endsWith('.toml')
+              ? toml.parse(data)
+              : json5.parse(data)
       }
     }
-  } else if (values) {
-    log.debug('Loading config from values.')
-    configSchema.load(values)
-  } else {
-    log.debug('Loading config from default values.')
-    configSchema.load({})
   }
-
-  configSchema.validate({ allowed: 'strict' })
-  const config = configSchema.getProperties()
-
-  log.debug('Using config:', config)
-  return config
+  if (!Array.isArray(values)) {
+    values = [values || {}]
+  }
+  for (const value of values) {
+    const schema = convict(configSchema)
+    schema.load(value || {})
+    schema.validate({ allowed: 'strict' })
+    configs.push(schema.getProperties())
+  }
+  log.debug('Using config:', configs)
+  return configs
 }
 
 function getFunctionDeclaration() {
   const properties: Record<string, { type: string; description: string; nullable?: boolean }> = {}
   const required: string[] = []
-  const schema = configSchema.getSchema()
+  const schema = convict(configSchema).getSchema()
 
   Object.entries(schema._cvtProperties).forEach(([name, value]) => {
     const { format, doc, nullable } = value as SchemaObj

package/src/docker.ts

@@ -0,0 +1,131 @@
+import path from 'path'
+import Docker from 'dockerode'
+import { logger, resolvePackagePath } from './utils'
+import { loadConfig } from './config'
+import fs from 'fs'
+
+const log = logger('webrtcperf:docker')
+
+export async function runWithDocker(argv: string[]) {
+  const docker = new Docker()
+  const configPath = argv.filter(s => s !== '--docker')[0]
+  if (!configPath) throw new Error('No configuration file specified')
+  const configName = path.basename(configPath)
+  const config = (await loadConfig(configPath))[0]
+
+  const startTimestamp = Date.now()
+  const dataDir = path.resolve(path.dirname(configPath), 'logs', `${startTimestamp}`)
+  await fs.promises.mkdir(dataDir, { recursive: true })
+
+  const binds: string[] = [
+    `${path.resolve(configPath)}:/config/${configName}:ro`,
+    '/dev/shm:/dev/shm',
+    `${dataDir}:/data`,
+    '/tmp/webrtcperf-cache:/root/.webrtcperf',
+  ]
+
+  if (config.scriptPath) {
+    const scriptName = path.basename(config.scriptPath)
+    binds.push(`${path.resolve(config.scriptPath)}:/scripts/${scriptName}:ro`)
+  }
+
+  if (process.env.DEBUG_SRC) {
+    binds.push(`${resolvePackagePath('app.min.js')}:/app/app.min.js:ro`)
+  }
+
+  const portBindings: Docker.PortMap = {}
+  const exposedPorts: { [portAndProtocol: string]: object } = {}
+  if (config.debuggingPort) {
+    for (let i = 0; i < config.sessions; i++) {
+      const port = `${config.debuggingPort + i}/tcp`
+      portBindings[port] = [{ HostPort: `${config.debuggingPort + i}` }]
+      exposedPorts[port] = {}
+    }
+  }
+
+  const env = [
+    `DEBUG_LEVEL=${process.env.DEBUG_LEVEL || 'info'}`,
+    'SHOW_PAGE_LOG=false',
+    'SHOW_STATS=false',
+    'SERVER_PORT=5000',
+    'SERVER_USE_HTTPS=true',
+    'SERVER_DATA=/data',
+    `START_TIMESTAMP=${startTimestamp}`,
+    `STATS_PATH=/data/stats.csv`,
+    `PAGE_LOG_PATH=/data/page.log`,
+    `DETAILED_STATS_PATH=/data/detailed-stats.csv`,
+  ]
+
+  if (config.scriptPath) {
+    const scriptName = path.basename(config.scriptPath)
+    env.push(`SCRIPT_PATH=/scripts/${scriptName}`)
+  }
+
+  if (config.debuggingPort) {
+    env.push(`DEBUGGING_PORT=${config.debuggingPort}`)
+  }
+
+  if (config.prometheusPushgateway.startsWith('http://localhost')) {
+    env.push('PROMETHEUS_PUSHGATEWAY=http://pushgateway:9091')
+  }
+
+  const containerConfig: Docker.ContainerCreateOptions = {
+    Image: 'ghcr.io/vpalmisano/webrtcperf:devel',
+    name: 'webrtcperf',
+    Cmd: [`/config/${configName}`],
+    HostConfig: {
+      Binds: binds,
+      PortBindings: portBindings,
+      CapAdd: ['NET_ADMIN'],
+      NetworkMode: config.prometheusPushgateway.startsWith('http://localhost') ? 'prometheus-stack_default' : 'bridge',
+    },
+    Env: env,
+    AttachStdin: true,
+    AttachStdout: true,
+    AttachStderr: true,
+    Tty: true,
+    OpenStdin: true,
+    StdinOnce: true,
+    ExposedPorts: exposedPorts,
+  }
+
+  try {
+    if (!process.env.DEBUG_SRC) {
+      log.info('Pulling latest development image...')
+      await docker.pull('ghcr.io/vpalmisano/webrtcperf:devel')
+    }
+
+    try {
+      const existingContainer = await docker.getContainer('webrtcperf')
+      await existingContainer.remove({ force: true })
+      // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    } catch (err: unknown) {
+      // Container doesn't exist, continue
+    }
+
+    const container = await docker.createContainer(containerConfig)
+    await container.start()
+
+    const stream = await container.attach({
+      stream: true,
+      stdin: true,
+      stdout: true,
+      stderr: true,
+    })
+
+    process.stdin.pipe(stream)
+    stream.pipe(process.stdout)
+
+    await new Promise(resolve => {
+      container.wait((err: Error, data: { StatusCode: number }) => {
+        if (err) log.error('Error waiting for container:', data, err.stack)
+        resolve(data)
+      })
+    })
+
+    await container.remove()
+  } catch (error) {
+    log.error('Docker operation failed:', error)
+    throw error
+  }
+}