karaoke-eternal 1.0.0 → 2.0.0-beta.6

package/build/server/Rooms/router.js

@@ -0,0 +1,97 @@
+import KoaRouter from '@koa/router';
+import sql from 'sqlate';
+import Database from '../lib/Database.js';
+import getLogger from '../lib/Log.js';
+import Rooms, { STATUSES } from '../Rooms/Rooms.js';
+import { ValidationError } from '../lib/Errors.js';
+const log = getLogger('Rooms');
+const { db } = Database;
+const router = new KoaRouter({ prefix: '/api/rooms' });
+import { ROOM_PREFS_PUSH } from '../../shared/actionTypes.js';
+// list rooms
+router.get('/:roomId?', async (ctx) => {
+    const roomId = ctx.params.roomId ? parseInt(ctx.params.roomId, 10) : undefined;
+    const status = ctx.user.isAdmin ? STATUSES : undefined;
+    const res = await Rooms.get(roomId, { status });
+    res.result.forEach((roomId) => {
+        if (ctx.user.isAdmin) {
+            const room = ctx.io.sockets.adapter.rooms.get(Rooms.prefix(roomId));
+            res.entities[roomId].numUsers = room ? room.size : 0;
+        }
+        else {
+            // only pass the 'roles' prefs key
+            res.entities[roomId].prefs = res.entities[roomId].prefs?.roles ? { roles: res.entities[roomId].prefs.roles } : {};
+        }
+    });
+    ctx.body = res;
+});
+// create room
+router.post('/', async (ctx) => {
+    if (!ctx.user.isAdmin) {
+        ctx.throw(401);
+    }
+    try {
+        const res = await Rooms.set(undefined, ctx.request.body);
+        log.verbose('%s created a room (roomId: %s)', ctx.user.name, res.lastID);
+    }
+    catch (err) {
+        if (err instanceof ValidationError)
+            ctx.throw(422, err.message);
+        throw err;
+    }
+    // send updated room list
+    ctx.body = await Rooms.get(null, { status: STATUSES });
+});
+// update room
+router.put('/:roomId', async (ctx) => {
+    if (!ctx.user.isAdmin) {
+        ctx.throw(401);
+    }
+    const roomId = parseInt(ctx.params.roomId, 10);
+    try {
+        await Rooms.set(roomId, ctx.request.body);
+    }
+    catch (err) {
+        if (err instanceof ValidationError)
+            ctx.throw(422, err.message);
+        throw err;
+    }
+    log.verbose('%s updated a room (roomId: %s)', ctx.user.name, roomId);
+    const sockets = await ctx.io.in(Rooms.prefix(roomId)).fetchSockets();
+    for (const s of sockets) {
+        if (s?.user.isAdmin) {
+            ctx.io.to(s.id).emit('action', {
+                type: ROOM_PREFS_PUSH,
+                payload: await Rooms.get(roomId),
+            });
+        }
+    }
+    // send updated room list
+    ctx.body = await Rooms.get(null, { status: STATUSES });
+});
+// remove room
+router.delete('/:roomId', async (ctx) => {
+    if (!ctx.user.isAdmin) {
+        ctx.throw(401);
+    }
+    const roomId = parseInt(ctx.params.roomId, 10);
+    if (typeof roomId !== 'number') {
+        ctx.throw(422, 'Invalid roomId');
+    }
+    // remove room's queue first
+    const queueQuery = sql `
+    DELETE FROM queue
+    WHERE roomId = ${roomId}
+  `;
+    await db.run(String(queueQuery), queueQuery.parameters);
+    // remove room
+    const roomQuery = sql `
+    DELETE FROM rooms
+    WHERE roomId = ${roomId}
+  `;
+    await db.run(String(roomQuery), roomQuery.parameters);
+    log.verbose('%s deleted roomId %s', ctx.user.name, roomId);
+    // send updated room list
+    ctx.body = await Rooms.get(null, { status: STATUSES });
+});
+export default router;

package/build/server/Rooms/socket.js

@@ -0,0 +1,23 @@
+import Rooms from './Rooms.js';
+import { ROOM_PREFS_PUSH_REQUEST, ROOM_PREFS_PUSH, _ERROR, } from '../../shared/actionTypes.js';
+const ACTION_HANDLERS = {
+    [ROOM_PREFS_PUSH_REQUEST]: async (sock, { payload }, acknowledge) => {
+        const { roomId } = payload;
+        if (!sock.user.isAdmin || !roomId) {
+            acknowledge({
+                type: ROOM_PREFS_PUSH_REQUEST + _ERROR,
+                error: 'Unauthorized',
+            });
+        }
+        const sockets = await sock.server.in(Rooms.prefix(roomId)).fetchSockets();
+        for (const s of sockets) {
+            if (s?.user.isAdmin) {
+                sock.server.to(s.id).emit('action', {
+                    type: ROOM_PREFS_PUSH,
+                    payload,
+                });
+            }
+        }
+    },
+};
+export default ACTION_HANDLERS;

package/build/server/Scanner/FileScanner/FileScanner.js

@@ -0,0 +1,166 @@
+import path from 'path';
+import fsPromises from 'node:fs/promises';
+import { parseBuffer } from 'music-metadata';
+import { unzip } from 'unzipit';
+import getLogger from '../../lib/Log.js';
+import { getExt } from '../../lib/util.js';
+import getFiles from './getFiles.js';
+import getConfig from './getConfig.js';
+import getCdgName from '../../lib/getCdgName.js';
+import Media from '../../Media/Media.js';
+import MetaParser from '../MetaParser/MetaParser.js';
+import Scanner from '../Scanner.js';
+import IPC from '../../lib/IPCBridge.js';
+import fileTypes from '../../Media/fileTypes.js';
+import { LIBRARY_MATCH_SONG, MEDIA_ADD, MEDIA_REMOVE, MEDIA_UPDATE } from '../../../shared/actionTypes.js';
+const log = getLogger('FileScanner');
+const audioExts = Object.keys(fileTypes).filter(ext => fileTypes[ext].mimeType.startsWith('audio/'));
+const searchExts = Object.keys(fileTypes).filter(ext => fileTypes[ext].scan !== false);
+class FileScanner extends Scanner {
+    paths;
+    parser;
+    constructor(prefs, qStats) {
+        super(qStats);
+        this.paths = prefs.paths;
+    }
+    async scan(pathId) {
+        const dir = this.paths.entities[pathId]?.path;
+        const validMediaIds = [];
+        let files; // { file, stats }[]
+        let prevDir;
+        if (!dir) {
+            log.error('invalid pathId: %s', pathId);
+            return;
+        }
+        log.info('Searching: %s', dir);
+        this.emitStatus(`Searching: ${dir}`, 0);
+        try {
+            files = await getFiles(dir, file => searchExts.includes(getExt(file)));
+            log.info('  => found %s files with valid extensions %s', files.length.toLocaleString(), JSON.stringify(searchExts));
+        }
+        catch (err) {
+            log.error(`  => ${err.message} (path offline)`);
+            return;
+        }
+        for (let i = 0; i < files.length; i++) {
+            const curDir = path.dirname(files[i].file);
+            if (prevDir !== curDir) {
+                prevDir = curDir;
+                // (re)init parser with this folder's config, if any
+                const cfg = getConfig(curDir, dir);
+                this.parser = new MetaParser(cfg);
+            }
+            log.info('[%s/%s] %s', i + 1, files.length, files[i].file);
+            this.emitStatus(`Processing (${i + 1} of ${files.length})`, (i + 1) / files.length);
+            // process file
+            try {
+                const res = await this.process(files[i], pathId);
+                validMediaIds.push(res.mediaId);
+            }
+            catch (err) {
+                log.warn(`  => ${err.message}`);
+            }
+            if (this.isCanceling) {
+                this.emitStatus('Stopped', 100, false);
+                return;
+            }
+        } // end for
+        log.info('Processed %s valid media files', validMediaIds.length.toLocaleString());
+        log.info('Searching for invalid media entries');
+        const numRemoved = await this.removeInvalid(pathId, validMediaIds);
+        log.info(`Removed ${numRemoved} invalid media entries`);
+    }
+    async process({ file }, pathId) {
+        let buffer = await fsPromises.readFile(file);
+        let mimeType = fileTypes[getExt(file)].mimeType;
+        if (getExt(file) === '.zip') {
+            const { entries } = await unzip(new Uint8Array(buffer));
+            const audioName = Object.keys(entries).find(f => !f.includes('/') && audioExts.includes(getExt(f)));
+            if (!audioName)
+                throw new Error(`no valid audio file ${JSON.stringify(audioExts)} found in archive`);
+            const cdgName = Object.keys(entries).find(f => !f.includes('/') && getExt(f) === '.cdg');
+            if (!cdgName)
+                throw new Error('no .cdg sidecar found in archive');
+            buffer = Buffer.from(await entries[audioName].arrayBuffer());
+            mimeType = fileTypes[getExt(audioName)].mimeType;
+        }
+        else {
+            if (fileTypes[getExt(file)].requiresCDG && !(await getCdgName(file)))
+                throw new Error('no .cdg sidecar found');
+        }
+        const data = await parseBuffer(buffer, mimeType, {
+            duration: true,
+            skipCovers: true,
+        });
+        if (!data.format.duration) {
+            throw new Error('could not determine duration');
+        }
+        log.verbose('  => duration: %s:%s', Math.floor(data.format.duration / 60), Math.round(data.format.duration % 60).toString().padStart(2, '0'));
+        // run MetaParser
+        const pathInfo = path.parse(file);
+        const parsed = this.parser({
+            dir: pathInfo.dir,
+            dirSep: path.sep,
+            name: pathInfo.name,
+            meta: data.common,
+        });
+        // get artistId and songId
+        const match = await IPC.req({ type: LIBRARY_MATCH_SONG, payload: parsed });
+        const media = {
+            songId: match.songId,
+            pathId,
+            // normalize relPath to forward slashes with no leading slash
+            relPath: file.substring(this.paths.entities[pathId].path.length).replace(/\\/g, '/').replace(/^\//, ''),
+            duration: Math.round(data.format.duration),
+            rgTrackGain: data.common.replaygain_track_gain ? data.common.replaygain_track_gain.dB : null,
+            rgTrackPeak: data.common.replaygain_track_peak ? data.common.replaygain_track_peak.ratio : null,
+        };
+        // file already in database?
+        const res = await Media.search({
+            pathId,
+            relPath: media.relPath,
+        });
+        log.verbose('  => %s db result(s)', res.result.length);
+        if (res.result.length) {
+            const row = res.entities[res.result[0]];
+            const diff = {};
+            // did anything change?
+            Object.keys(media).forEach((key) => {
+                if (media[key] !== row[key])
+                    diff[key] = media[key];
+            });
+            if (Object.keys(diff).length) {
+                await IPC.req({
+                    type: MEDIA_UPDATE,
+                    payload: {
+                        mediaId: row.mediaId,
+                        dateUpdated: Math.round(new Date().getTime() / 1000), // seconds
+                        ...diff,
+                    },
+                });
+                log.info('  => updated: %s', Object.keys(diff).join(', '));
+            }
+            else {
+                log.info('  => ok');
+            }
+            return { mediaId: row.mediaId, isNew: false };
+        } // end if
+        // new media
+        ;
+        media.dateAdded = Math.round(new Date().getTime() / 1000); // seconds
+        log.info('  => new: %s', JSON.stringify(match));
+        return {
+            mediaId: await IPC.req({ type: MEDIA_ADD, payload: media }),
+            isNew: true,
+        };
+    }
+    async removeInvalid(pathId, validMediaIds = []) {
+        const res = await Media.search({ pathId });
+        const invalid = res.result.filter(mediaId => !validMediaIds.includes(mediaId));
+        if (invalid.length) {
+            await IPC.req({ type: MEDIA_REMOVE, payload: invalid });
+        }
+        return invalid.length;
+    }
+}
+export default FileScanner;

package/build/server/Scanner/FileScanner/getConfig.js

@@ -0,0 +1,32 @@
+import path from 'path';
+import getLogger from '../../lib/Log.js';
+import fs from 'fs';
+import JSON5 from 'json5';
+const log = getLogger('FileScanner');
+const CONFIG = '_kes.v2.json';
+// search each folder from dir up to baseDir
+function getConfig(dir, baseDir) {
+    dir = path.normalize(dir);
+    baseDir = path.normalize(baseDir);
+    const cfgPath = path.resolve(dir, CONFIG);
+    try {
+        const userScript = fs.readFileSync(cfgPath, 'utf-8');
+        log.info('Using custom parser config: %s', cfgPath);
+        try {
+            return JSON5.parse(userScript);
+        }
+        catch (err) {
+            log.error(err);
+        }
+    }
+    catch {
+        log.verbose('No parser config found: %s', dir);
+    }
+    if (dir === baseDir) {
+        log.info('Using default parser config');
+        return;
+    }
+    // try parent dir
+    return getConfig(path.resolve(dir, '..'), baseDir);
+}
+export default getConfig;

package/build/server/Scanner/FileScanner/getFiles.js

@@ -0,0 +1,61 @@
+import fs from 'fs';
+import path from 'path';
+import getLogger from '../../lib/Log.js';
+const log = getLogger('FileScanner:getFiles');
+/**
+ * Silly promise wrapper for synchronous walker
+ *
+ * We want a synchronous walker for performance, but FileScanner runs
+ * the walker in a loop, which will block the (async) socket.io status
+ * emissions unless we use setTimeout here. @todo is there a better way?
+ *
+ * @param  {string} dir      path to recursively list
+ * @param  {function} filterFn filter function applied to each file
+ * @return {array}          array of objects with path and stat properties
+ */
+function getFiles(dir, filterFn) {
+    return new Promise((resolve, reject) => {
+        setTimeout(() => {
+            try {
+                resolve(walkSync(dir, filterFn));
+            }
+            catch (err) {
+                reject(err);
+            }
+        }, 0);
+    });
+}
+/**
+ * Directory walker that only throws if parent directory
+ * can't be read. Errors stat-ing children are only logged.
+ */
+function walkSync(dir, filterFn) {
+    let results = [];
+    const list = fs.readdirSync(dir);
+    list.forEach((file) => {
+        let stats;
+        file = path.join(dir, file);
+        try {
+            stats = fs.statSync(file);
+        }
+        catch (err) {
+            log.warn(err.message);
+            return;
+        }
+        if (stats && stats.isDirectory()) {
+            try {
+                results = results.concat(walkSync(file, filterFn));
+            }
+            catch (err) {
+                log.warn(err.message);
+            }
+        }
+        else {
+            if (!filterFn || filterFn(file)) {
+                results.push({ file, stats });
+            }
+        }
+    });
+    return results;
+}
+export default getFiles;

package/build/server/Scanner/MetaParser/MetaParser.js

@@ -0,0 +1,77 @@
+import getLogger from '../../lib/Log.js';
+import { composeSync } from 'ctx-compose';
+import jsone from 'json-e';
+import defaultMiddleware from './defaultMiddleware.js';
+const log = getLogger('MetaParser');
+const defaultParser = compose(...defaultMiddleware.values());
+const parserCfgProps = ['articles', 'artistOnLeft', 'delimiter'];
+function compose(...args) {
+    const flattened = args.reduce((accumulator, currentValue) => accumulator.concat(currentValue), []);
+    return composeSync(flattened);
+}
+const customFunctions = {
+    replace: (predicate, search, ...args) => {
+        return args.length === 1
+            ? predicate.replace(search, args[0])
+            : predicate.replace(new RegExp(search, args[0]), args[1]);
+    },
+};
+// default parser creator
+function getDefaultParser(cfg = {}) {
+    if (typeof cfg.articles === 'undefined') {
+        cfg.articles = ['A', 'An', 'The'];
+    }
+    return (ctx, next) => {
+        Object.assign(ctx.cfg, cfg);
+        return defaultParser(ctx, next);
+    };
+}
+class MetaParser {
+    constructor(userCfg = {}) {
+        const parserCfg = {};
+        const template = {};
+        // we accept parser config and JSON-e template items (both
+        // user-supplied) in a flat object format; separate them here
+        for (const [key, val] of Object.entries(userCfg)) {
+            if (parserCfgProps.includes(key))
+                parserCfg[key] = val;
+            else
+                template[key] = val;
+        }
+        const parser = getDefaultParser(parserCfg);
+        const isUserTemplate = !!Object.keys(template).length;
+        return (scannerCtx) => {
+            let ctx = {
+                cfg: parserCfg,
+                ...scannerCtx,
+            };
+            parser(ctx, () => { });
+            if (isUserTemplate) {
+                const res = jsone(template, { ...ctx, ...customFunctions });
+                Object.keys(res).forEach((key) => {
+                    if (typeof res[key] === 'string')
+                        res[key] = res[key].trim();
+                });
+                if (res.artist)
+                    res.artistNorm = res.artistNorm ?? res.artist;
+                if (res.title)
+                    res.titleNorm = res.titleNorm ?? res.title;
+                log.debug('User template:');
+                log.debug(template);
+                log.debug('Result:');
+                log.debug(res);
+                ctx = { ...ctx, ...res };
+            }
+            if (!ctx.artist || !ctx.title) {
+                throw new Error('could not determine artist or title');
+            }
+            return {
+                artist: ctx.artist,
+                artistNorm: ctx.artistNorm ?? ctx.artist,
+                title: ctx.title,
+                titleNorm: ctx.titleNorm ?? ctx.title,
+            };
+        };
+    }
+}
+export default MetaParser;

package/build/server/Scanner/MetaParser/defaultMiddleware.js

@@ -0,0 +1,170 @@
+const m = new Map();
+export default m;
+// ----------------------
+// begin middleware stack
+// ----------------------
+m.set('normalize whitespace', (ctx, next) => {
+    ctx.name = ctx.name.replace(/_/g, ' '); // underscores to spaces
+    ctx.name = ctx.name.replace(/ {2,}/g, ' '); // multiple spaces to single
+    next();
+});
+m.set('de-karaoke', (ctx, next) => {
+    // 'karaoke' or 'vocal' surrounded by (), [], or {}
+    ctx.name = ctx.name.replace(/[([{](?=[^([{]*$).*(?:karaoke|vocal).*[)\]}]/i, '');
+    next();
+});
+// --------------
+// parse
+// --------------
+// detect delimiter and split to parts
+m.set('split', (ctx, next) => {
+    const inTheStyleOf = ctx.name.match(/ in the style of /i);
+    ctx.cfg = {
+        delimiter: inTheStyleOf ? inTheStyleOf[0] : '-',
+        artistOnLeft: !inTheStyleOf,
+        ...ctx.cfg,
+    };
+    // allow leading and/or trailing space when searching for delimiter,
+    // then pick the match with the most whitespace (longest match) as it's
+    // most likely to be the actual delimiter rather than a false positive
+    const d = ctx.cfg.delimiter instanceof RegExp ? ctx.cfg.delimiter : new RegExp(` ?${ctx.cfg.delimiter} ?`, 'g');
+    const matches = ctx.name.match(d);
+    if (!matches) {
+        throw new Error('no artist/title delimiter in filename');
+    }
+    const longest = matches.reduce((a, b) => a.length > b.length ? a : b);
+    ctx.parts = ctx.name.split(longest);
+    if (ctx.parts.length < 2) {
+        throw new Error('no artist/title delimiter in filename');
+    }
+    next();
+});
+m.set('clean parts', cleanParts([
+    /^\d*\.?$/, // looks like a track number
+    /^\W*$/, // all non-word chars
+    /^[a-zA-Z]{2,4}[ -]?\d{1,}/i, // 2-4 letters followed by 1 or more digits
+]));
+// set title
+m.set('set title', (ctx, next) => {
+    // skip if already set
+    if (ctx.title)
+        return next();
+    // @todo this assumes delimiter won't appear in title
+    ctx.title = ctx.cfg.artistOnLeft ? ctx.parts.pop() : ctx.parts.shift();
+    ctx.title = ctx.title.trim();
+    next();
+});
+// set artist
+m.set('set artist', (ctx, next) => {
+    // skip if already set
+    if (ctx.artist)
+        return next();
+    ctx.artist = ctx.parts.join(ctx.cfg.delimiter);
+    ctx.artist = ctx.artist.trim();
+    next();
+});
+// -----------
+// post
+// -----------
+// remove any surrounding quotes
+m.set('remove quotes', (ctx, next) => {
+    ctx.artist = ctx.artist.replace(/^['|"](.*)['|"]$/, '$1');
+    ctx.title = ctx.title.replace(/^['|"](.*)['|"]$/, '$1');
+    next();
+});
+// some artist-specific tweaks
+m.set('artist tweaks', (ctx, next) => {
+    // Last, First [Middle] -> First [Middle] Last
+    ctx.artist = ctx.artist.replace(/^(\w+), (\w+ ?\w+)$/ig, '$2 $1');
+    // featuring/feat/ft -> ft.
+    ctx.artist = ctx.artist.replace(/ featuring /i, ' ft. ');
+    ctx.artist = ctx.artist.replace(/ f(ea)?t\.? /i, ' ft. ');
+    next();
+});
+// move leading articles to end
+m.set('move leading articles', (ctx, next) => {
+    ctx.artist = moveArticles(ctx.artist, ctx.cfg.articles);
+    ctx.title = moveArticles(ctx.title, ctx.cfg.articles);
+    next();
+});
+// ---------
+// normalize
+// ---------
+m.set('normalize artist', (ctx, next) => {
+    // skip if already set
+    if (ctx.artistNorm)
+        return next();
+    ctx.artistNorm = normalizeStr(ctx.artist, ctx.cfg.articles);
+    next();
+});
+m.set('normalize title', (ctx, next) => {
+    // skip if already set
+    if (ctx.titleNorm)
+        return next();
+    ctx.titleNorm = normalizeStr(ctx.title, ctx.cfg.articles);
+    next();
+});
+// ---------------------
+// end middleware stack
+// ---------------------
+// clean left-to-right until a valid part is encountered (or only 2 parts left)
+function cleanParts(patterns) {
+    return function (ctx, next) {
+        for (let i = 0; i < ctx.parts.length; i++) {
+            if (patterns.some(exp => exp.test(ctx.parts[i].trim())) && ctx.parts.length > 2) {
+                ctx.parts.shift();
+                i--;
+            }
+            else
+                break;
+        }
+        next();
+    };
+}
+function normalizeStr(str, articles) {
+    str = removeArticles(str, articles)
+        .normalize('NFD')
+        .replace(/[\u0300-\u036f]/g, '')
+        .replace(' & ', ' and ') // normalize ampersand
+        .replace(/[^\w\s]|_/g, ''); // remove punctuation
+    return str;
+}
+// move leading articles to end (but before any parentheses)
+function moveArticles(str, articles) {
+    if (!Array.isArray(articles))
+        return str;
+    for (const article of articles) {
+        const search = article + ' ';
+        // leading article?
+        if (new RegExp(`^${search}`, 'i').test(str)) {
+            const parens = /[([{].*$/.exec(str);
+            if (parens) {
+                str = str.substring(search.length, parens.index - search.length)
+                    .trim() + `, ${article} ${parens[0]}`;
+            }
+            else {
+                str = str.substring(search.length) + `, ${article}`;
+            }
+            // only replace one article per string
+            continue;
+        }
+    }
+    return str.trim();
+}
+function removeArticles(str, articles) {
+    if (!Array.isArray(articles))
+        return str;
+    for (const article of articles) {
+        const leading = new RegExp(`^${article} `, 'i');
+        const trailing = new RegExp(`, ${article}$`, 'i');
+        if (leading.test(str)) {
+            str = str.replace(leading, '');
+            continue; // only replace one article per string
+        }
+        else if (trailing.test(str)) {
+            str = str.replace(trailing, '');
+            continue; // only replace one article per string
+        }
+    }
+    return str.trim();
+}

package/build/server/Scanner/Scanner.js

@@ -0,0 +1,26 @@
+import IPC from '../lib/IPCBridge.js';
+import { SCANNER_WORKER_STATUS } from '../../shared/actionTypes.js';
+class Scanner {
+    isCanceling;
+    emitStatus;
+    constructor(qStats) {
+        this.isCanceling = false;
+        this.emitStatus = this.getStatusEmitter(qStats);
+    }
+    cancel() {
+        this.isCanceling = true;
+    }
+    getStatusEmitter({ length }) {
+        return (text, progress, isScanning = true) => {
+            IPC.send({
+                type: SCANNER_WORKER_STATUS,
+                payload: {
+                    isScanning,
+                    pct: (progress / length) * 100,
+                    text: length === 1 ? text : `[1/${length}] ${text}`,
+                },
+            });
+        };
+    }
+}
+export default Scanner;