karaoke-eternal 1.0.0

package/server/Scanner/FileScanner/FileScanner.js

@@ -0,0 +1,225 @@
+const path = require('path')
+const log = require('../../lib/Log')('FileScanner')
+const musicMeta = require('music-metadata')
+const getFiles = require('./getFiles')
+const getConfig = require('./getConfig')
+const getPerms = require('../../lib/getPermutations')
+const getCdgName = require('../../lib/getCdgName')
+const Media = require('../../Media')
+const MetaParser = require('../MetaParser')
+const Scanner = require('../Scanner')
+const IPC = require('../../lib/IPCBridge')
+const {
+  LIBRARY_MATCH_SONG,
+  MEDIA_ADD,
+  MEDIA_CLEANUP,
+  MEDIA_REMOVE,
+  MEDIA_UPDATE,
+} = require('../../../shared/actionTypes')
+
+const searchExts = ['mp4', 'm4a', 'mp3']
+const searchExtPerms = searchExts.reduce((perms, ext) => perms.concat(getPerms(ext)), [])
+
+class FileScanner extends Scanner {
+  constructor (prefs) {
+    super()
+    this.paths = prefs.paths
+  }
+
+  async scan () {
+    const files = new Map() // pathId => [files]
+    const valid = [] // mediaIds
+    let i = 0 // file counter
+    let numTotal = 0
+    let numAdded = 0
+
+    // get list of files from all paths
+    for (const pathId of this.paths.result) {
+      const basePath = this.paths.entities[pathId].path
+
+      this.emitStatus(`Listing folder ${this.paths.result.indexOf(pathId) + 1} of ${this.paths.result.length}`, 0)
+      log.info('Searching path: %s', basePath)
+
+      try {
+        const list = await getFiles(basePath, file => searchExtPerms.some(ext => file.endsWith('.' + ext)))
+        files.set(pathId, list)
+        numTotal += list.length
+
+        log.info('  => found %s files with valid extensions %s',
+          list.length.toLocaleString(),
+          JSON.stringify(searchExts)
+        )
+      } catch (err) {
+        log.error(`  => ${err.message} (path offline)`)
+      }
+
+      if (this.isCanceling) {
+        return
+      }
+    } // end for
+
+    log.info('Processing %s files', numTotal.toLocaleString())
+
+    for (const [pathId, list] of files) {
+      let lastDir
+
+      for (const item of list) {
+        i++
+        log.info('[%s/%s] %s', i, numTotal, item.file)
+        this.emitStatus(`Scanning media (${i.toLocaleString()} of ${numTotal.toLocaleString()})`, (i / numTotal) * 100)
+
+        const dir = path.dirname(item.file)
+
+        if (lastDir !== dir) {
+          lastDir = dir
+
+          // (re)init parser with this folder's config, if any
+          const cfg = getConfig(dir, this.paths.entities[pathId].path)
+          this.parser = new MetaParser(cfg)
+        }
+
+        // process file
+        try {
+          const res = await this.process(item, pathId)
+
+          // success
+          valid.push(res.mediaId)
+          if (res.isNew) numAdded++
+        } catch (err) {
+          log.warn(`  => ${err.message}: ${item.file}`)
+        }
+
+        if (this.isCanceling) {
+          this.emitStatus(`Stopped (${numAdded} new media)`, 100, false)
+          return
+        }
+      } // end for
+    } // end for
+
+    log.info('Processed %s valid media files', valid.length.toLocaleString())
+
+    const numRemoved = await this.removeInvalid(valid, Array.from(files.keys()))
+
+    this.emitStatus(`Finished (${numAdded} new, ${numRemoved} removed, ${valid.length} total media)`, 100, false)
+
+    await IPC.req({ type: MEDIA_CLEANUP })
+  }
+
+  async process (item, pathId) {
+    const data = await musicMeta.parseFile(item.file, {
+      duration: true,
+      skipCovers: true,
+    })
+
+    if (!data.format.duration) {
+      throw new Error('could not determine duration')
+    }
+
+    log.verbose('  => duration: %s:%s',
+      Math.floor(data.format.duration / 60),
+      Math.round(data.format.duration % 60, 10).toString().padStart(2, '0')
+    )
+
+    // run MetaParser
+    const pathInfo = path.parse(item.file)
+    const parsed = this.parser({
+      dir: pathInfo.dir,
+      dirSep: path.sep,
+      name: pathInfo.name,
+      data: data.common,
+    })
+
+    // get artistId and songId
+    const match = await IPC.req({ type: LIBRARY_MATCH_SONG, payload: parsed })
+
+    const media = {
+      songId: match.songId,
+      pathId,
+      // normalize relPath to forward slashes with no leading slash
+      relPath: item.file.substring(this.paths.entities[pathId].path.length).replace(/\\/g, '/').replace(/^\//, ''),
+      duration: Math.round(data.format.duration),
+      rgTrackGain: data.common.replaygain_track_gain ? data.common.replaygain_track_gain.dB : null,
+      rgTrackPeak: data.common.replaygain_track_peak ? data.common.replaygain_track_peak.ratio : null,
+    }
+
+    // need to look for .cdg if this is an audio-only file
+    if (!/\.mp4/i.test(path.extname(item.file))) {
+      if (!await getCdgName(item.file)) {
+        throw new Error('no .cdg sidecar found; skipping')
+      }
+
+      log.verbose('  => found .cdg sidecar')
+    }
+
+    // file already in database?
+    const res = await Media.search({
+      pathId,
+      relPath: media.relPath,
+    })
+
+    log.verbose('  => %s db result(s)', res.result.length)
+
+    if (res.result.length) {
+      const row = res.entities[res.result[0]]
+      const diff = {}
+
+      // did anything change?
+      Object.keys(media).forEach(key => {
+        if (media[key] !== row[key]) diff[key] = media[key]
+      })
+
+      if (Object.keys(diff).length) {
+        await IPC.req({
+          type: MEDIA_UPDATE,
+          payload: {
+            mediaId: row.mediaId,
+            dateUpdated: Math.round(new Date().getTime() / 1000), // seconds
+            ...diff,
+          }
+        })
+
+        log.info('  => updated: %s', Object.keys(diff).join(', '))
+      } else {
+        log.info('  => ok')
+      }
+
+      return { mediaId: row.mediaId, isNew: false }
+    } // end if
+
+    // new media
+    // ---------
+    media.dateAdded = Math.round(new Date().getTime() / 1000) // seconds
+    log.info('  => new: %s', JSON.stringify(match))
+
+    return {
+      mediaId: await IPC.req({ type: MEDIA_ADD, payload: media }),
+      isNew: true
+    }
+  }
+
+  async removeInvalid (validMedia = [], validPaths = []) {
+    log.info('Searching for invalid media entries')
+
+    const res = await Media.search()
+    const invalidMedia = []
+
+    res.result.forEach(mediaId => {
+      // just validated or in an offline path?
+      if (validMedia.includes(mediaId) || !validPaths.includes(res.entities[mediaId].pathId)) {
+        return
+      }
+
+      invalidMedia.push(mediaId)
+    })
+
+    log.info(`Found ${invalidMedia.length} invalid media entries`)
+
+    if (invalidMedia.length) {
+      await IPC.req({ type: MEDIA_REMOVE, payload: invalidMedia })
+    }
+
+    return invalidMedia.length
+  }
+}
+
+module.exports = FileScanner

package/server/Scanner/FileScanner/getConfig.js

@@ -0,0 +1,35 @@
+const path = require('path')
+const log = require('../../lib/Log')('FileScanner')
+const fs = require('fs')
+const { NodeVM } = require('vm2')
+const CONFIG = '_kes.v1.js'
+
+// search each folder from dir up to baseDir
+function getConfig (dir, baseDir) {
+  dir = path.normalize(dir)
+  baseDir = path.normalize(baseDir)
+  const cfgPath = path.resolve(dir, CONFIG)
+
+  try {
+    const userScript = fs.readFileSync(cfgPath, 'utf-8')
+    log.info('  => using config: %s', cfgPath)
+
+    try {
+      const vm = new NodeVM({ wrapper: 'none' })
+      return vm.run(userScript)
+    } catch (err) {
+      log.error(err)
+    }
+  } catch (err) {
+    log.verbose(`  => no config in ${dir}`)
+  }
+
+  if (dir === baseDir) {
+    return
+  }
+
+  // try parent dir
+  return getConfig(path.resolve(dir, '..'), baseDir)
+}
+
+module.exports = getConfig

package/server/Scanner/FileScanner/getFiles.js

@@ -0,0 +1,63 @@
+const fs = require('fs')
+const path = require('path')
+const log = require('../../lib/Log')('FileScanner:getFiles')
+
+/**
+ * Silly promise wrapper for synchronous walker
+ *
+ * We want a synchronous walker for performance, but FileScanner runs
+ * the walker in a loop, which will block the (async) socket.io status
+ * emissions unless we use setTimeout here. @todo is there a better way?
+ *
+ * @param  {string} dir      path to recursively list
+ * @param  {function} filterFn filter function applied to each file
+ * @return {array}          array of objects with path and stat properties
+ */
+function getFiles (dir, filterFn) {
+  return new Promise((resolve, reject) => {
+    setTimeout(() => {
+      try {
+        resolve(walkSync(dir, filterFn))
+      } catch (err) {
+        reject(err)
+      }
+    }, 0)
+  })
+}
+
+/**
+ * Directory walker that only throws if parent directory
+ * can't be read. Errors stat-ing children are only logged.
+ */
+function walkSync (dir, filterFn) {
+  let results = []
+  const list = fs.readdirSync(dir)
+
+  list.forEach(file => {
+    let stats
+    file = path.join(dir, file)
+
+    try {
+      stats = fs.statSync(file)
+    } catch (err) {
+      log.warn(err.message)
+      return
+    }
+
+    if (stats && stats.isDirectory()) {
+      try {
+        results = results.concat(walkSync(file, filterFn))
+      } catch (err) {
+        log.warn(err.message)
+      }
+    } else {
+      if (!filterFn || filterFn(file)) {
+        results.push({ file, stats })
+      }
+    }
+  })
+
+  return results
+}
+
+module.exports = getFiles

package/server/Scanner/FileScanner/index.js

@@ -0,0 +1,3 @@
+const FileScanner = require('./FileScanner')
+
+module.exports = FileScanner

package/server/Scanner/MetaParser/MetaParser.js

@@ -0,0 +1,49 @@
+const { composeSync } = require('ctx-compose')
+const defaultMiddleware = require('./defaultMiddleware')
+const defaultParser = compose(...defaultMiddleware.values())
+
+function compose (...args) {
+  const flattened = args.reduce(
+    (accumulator, currentValue) => accumulator.concat(currentValue), []
+  )
+
+  return composeSync(flattened)
+}
+
+// default parser creator
+function getDefaultParser (cfg = {}) {
+  if (typeof cfg.articles === 'undefined') {
+    cfg.articles = ['A', 'An', 'The']
+  }
+
+  return (ctx, next) => {
+    Object.assign(ctx.cfg, cfg)
+    return defaultParser(ctx, next)
+  }
+}
+
+class MetaParser {
+  constructor (cfg = {}) {
+    const parser = typeof cfg === 'function'
+      ? cfg({ compose, getDefaultParser, defaultMiddleware })
+      : getDefaultParser(cfg)
+
+    return input => {
+      const ctx = { ...input, cfg }
+      parser(ctx)
+
+      if (!ctx.artist || !ctx.title) {
+        throw new Error('could not determine artist or title')
+      }
+
+      return {
+        artist: ctx.artist,
+        artistNorm: ctx.artistNorm || ctx.artist,
+        title: ctx.title,
+        titleNorm: ctx.titleNorm || ctx.title,
+      }
+    }
+  }
+}
+
+module.exports = MetaParser

package/server/Scanner/MetaParser/defaultMiddleware.js

@@ -0,0 +1,197 @@
+const m = module.exports = new Map()
+
+// ----------------------
+// begin middleware stack
+// ----------------------
+
+m.set('normalize whitespace', (ctx, next) => {
+  ctx.name = ctx.name.replace(/_/g, ' ') // underscores to spaces
+  ctx.name = ctx.name.replace(/ {2,}/g, ' ') // multiple spaces to single
+  next()
+})
+
+m.set('de-karaoke', (ctx, next) => {
+  // 'karaoke' or 'vocal' surrounded by (), [], or {}
+  ctx.name = ctx.name.replace(/[([{](?=[^([{]*$).*(?:karaoke|vocal).*[)\]}]/i, '')
+  next()
+})
+
+// --------------
+// parse
+// --------------
+
+// detect delimiter and split to parts
+m.set('split', (ctx, next) => {
+  const inTheStyleOf = ctx.name.match(/ in the style of /i)
+
+  ctx.cfg = {
+    delimiter: inTheStyleOf ? inTheStyleOf[0] : '-',
+    artistOnLeft: !inTheStyleOf,
+    ...ctx.cfg,
+  }
+
+  // allow leading and/or trailing space when searching for delimiter,
+  // then pick the match with the most whitespace (longest match) as it's
+  // most likely to be the actual delimiter rather than a false positive
+  const d = ctx.cfg.delimiter instanceof RegExp ? ctx.cfg.delimiter : new RegExp(` ?${ctx.cfg.delimiter} ?`, 'g')
+  const matches = ctx.name.match(d)
+
+  if (!matches) {
+    throw new Error('no artist/title delimiter in filename')
+  }
+
+  const longest = matches.reduce((a, b) => a.length > b.length ? a : b)
+  ctx.parts = ctx.name.split(longest)
+
+  if (ctx.parts.length < 2) {
+    throw new Error('no artist/title delimiter in filename')
+  }
+
+  next()
+})
+
+m.set('clean parts', cleanParts([
+  /^\d*\.?$/, // looks like a track number
+  /^\W*$/, // all non-word chars
+  /^[a-zA-Z]{2,4}[ -]?\d{1,}/i, // 2-4 letters followed by 1 or more digits
+]))
+
+// set title
+m.set('set title', (ctx, next) => {
+  // skip if already set
+  if (ctx.title) return next()
+
+  // @todo this assumes delimiter won't appear in title
+  ctx.title = ctx.cfg.artistOnLeft ? ctx.parts.pop() : ctx.parts.shift()
+  ctx.title = ctx.title.trim()
+  next()
+})
+
+// set arist
+m.set('set artist', (ctx, next) => {
+  // skip if already set
+  if (ctx.artist) return next()
+
+  ctx.artist = ctx.parts.join(ctx.cfg.delimiter)
+  ctx.artist = ctx.artist.trim()
+  next()
+})
+
+// -----------
+// post
+// -----------
+
+// remove any surrounding quotes
+m.set('remove quotes', (ctx, next) => {
+  ctx.artist = ctx.artist.replace(/^['|"](.*)['|"]$/, '$1')
+  ctx.title = ctx.title.replace(/^['|"](.*)['|"]$/, '$1')
+  next()
+})
+
+// some artist-specific tweaks
+m.set('artist tweaks', (ctx, next) => {
+  // Last, First [Middle] -> First [Middle] Last
+  ctx.artist = ctx.artist.replace(/^(\w+), (\w+ ?\w+)$/ig, '$2 $1')
+
+  // featuring/feat/ft -> ft.
+  ctx.artist = ctx.artist.replace(/ featuring /i, ' ft. ')
+  ctx.artist = ctx.artist.replace(/ f(ea)?t\.? /i, ' ft. ')
+  next()
+})
+
+// move leading articles to end
+m.set('move leading articles', (ctx, next) => {
+  ctx.artist = moveArticles(ctx.artist, ctx.cfg.articles)
+  ctx.title = moveArticles(ctx.title, ctx.cfg.articles)
+  next()
+})
+
+// ---------
+// normalize
+// ---------
+m.set('normalize artist', (ctx, next) => {
+  // skip if already set
+  if (ctx.artistNorm) return next()
+
+  ctx.artistNorm = normalizeStr(ctx.artist, ctx.cfg.articles)
+  next()
+})
+
+m.set('normalize title', (ctx, next) => {
+  // skip if already set
+  if (ctx.titleNorm) return next()
+
+  ctx.titleNorm = normalizeStr(ctx.title, ctx.cfg.articles)
+  next()
+})
+
+// ---------------------
+// end middleware stack
+// ---------------------
+
+// clean left-to-right until a valid part is encountered (or only 2 parts left)
+function cleanParts (patterns) {
+  return function (ctx, next) {
+    for (let i = 0; i < ctx.parts.length; i++) {
+      if (patterns.some(exp => exp.test(ctx.parts[i].trim())) && ctx.parts.length > 2) {
+        ctx.parts.shift()
+        i--
+      } else break
+    }
+
+    next()
+  }
+}
+
+function normalizeStr (str, articles) {
+  str = removeArticles(str, articles)
+    .normalize('NFD')
+    .replace(/[\u0300-\u036f]/g, '')
+    .replace(' & ', ' and ') // normalize ampersand
+    .replace(/[^\w\s]|_/g, '') // remove punctuation
+
+  return str
+}
+
+// move leading articles to end (but before any parantheses)
+function moveArticles (str, articles) {
+  if (!Array.isArray(articles)) return str
+
+  for (const article of articles) {
+    const search = article + ' '
+
+    // leading article?
+    if (new RegExp(`^${search}`, 'i').test(str)) {
+      const parens = /[([{].*$/.exec(str)
+
+      if (parens) {
+        str = str.substring(search.length, parens.index - search.length)
+          .trim() + `, ${article} ${parens[0]}`
+      } else {
+        str = str.substring(search.length) + `, ${article}`
+      }
+
+      // only replace one article per string
+      continue
+    }
+  }
+
+  return str.trim()
+}
+
+function removeArticles (str, articles) {
+  for (const article of articles) {
+    const leading = new RegExp(`^${article} `, 'i')
+    const trailing = new RegExp(`, ${article}$`, 'i')
+
+    if (leading.test(str)) {
+      str = str.replace(leading, '')
+      continue // only replace one article per string
+    } else if (trailing.test(str)) {
+      str = str.replace(trailing, '')
+      continue // only replace one article per string
+    }
+  }
+
+  return str.trim()
+}

package/server/Scanner/MetaParser/index.js

@@ -0,0 +1,3 @@
+const MetaParser = require('./MetaParser')
+
+module.exports = MetaParser

package/server/Scanner/Scanner.js

@@ -0,0 +1,33 @@
+const IPC = require('../lib/IPCBridge')
+const {
+  SCANNER_WORKER_STATUS,
+} = require('../../shared/actionTypes')
+
+class Scanner {
+  constructor () {
+    this.isCanceling = false
+    this.emitStatus = this.getStatusEmitter()
+  }
+
+  cancel () {
+    this.isCanceling = true
+  }
+
+  getStatusEmitter () {
+    return (text, pct, isScanning = true) => {
+      IPC.send({
+        type: SCANNER_WORKER_STATUS,
+        payload: {
+          isScanning,
+          pct,
+          text,
+        },
+        meta: {
+          noAck: true,
+        }
+      })
+    }
+  }
+}
+
+module.exports = Scanner