@makefully/dutifully 1.10.1

package/src/classes/LipSync.js

@@ -0,0 +1,133 @@
+const
+    Cheer = require('./Cheer'),
+    fs = require('fs').promises,
+    getJSON = require('../helpers/getJSON'),
+    id3 = require('node-id3').Promise,
+    filter = (fileType, file) => (file.indexOf('.') !== 0) && (file.slice(-(fileType.length + 1)) === `.${fileType}`),
+    combine = async (path, fileType) => {
+        const
+            files = (await fs.readdir(path)).filter(filter.bind(null, fileType)),
+            list = [];
+        
+        for (let i = 0; i < files.length; i++) {
+            const
+                file = files[i];
+
+            if (fileType === 'mp3') { // check id3 for lyrics presence.
+                const
+                    {synchronisedLyrics} = await id3.read(`${path}${file}`);
+
+                if (synchronisedLyrics) {
+                    const
+                        index = synchronisedLyrics.map(({shortText}) => shortText).indexOf('lipsync');
+
+                    if (index >= 0) {
+                        list.push(file.substring(0, file.length - fileType.length - 1));
+                    }
+                }
+            } else {
+                list.push(file.substring(0, file.length - fileType.length - 1));
+            }
+        }
+
+        return list;
+    },
+    sortKeys = (obj) => Object.keys(obj).sort().reduce((s, key) => {
+        s[key] = obj[key];
+        return s;
+    }, {}),
+    mapReduction = (map, list) => list.reduce((obj, file) => {
+        const
+            id = file.substring(0, file.length - 4);
+
+        obj[id] = map[id];
+        return obj;
+    }, {}),
+    LipSync = class LipSync extends Cheer {
+        async prepare (data) {
+            this.differenceOnly = false;
+            this.updateList = null;
+            await this.replaceConfigPathWithJSON('script', 'files');
+            return super.prepare(data);
+        }
+
+        async checkDifference () {
+            const
+                {config} = this,
+                {files = {}, options = {}, output, src} = config,
+                {exportFormat = 'json', output: outputType = 'json'} = options,
+                file = 'mouthCues.json',
+                raw = outputType === 'json' && exportFormat !== 'mp3' ? await getJSON(`${output}${file}`) ?? {} : null,
+                alreadyLipSynced = raw ? Object.keys(raw) : await combine(output, exportFormat),
+                check = (id) => {
+                    const
+                        index = alreadyLipSynced.indexOf(id);
+
+                    if (index >= 0) {
+                        alreadyLipSynced.splice(index, 1);
+                        return true;
+                    } else {
+                        return false;
+                    }
+                },
+                mergeLipSync = async (newLipSync = false) => { // Must run _after_ all checks and will remove what's left.
+                    for (let i = 0; i < alreadyLipSynced.length; i++) {
+                        const
+                            key = alreadyLipSynced[i];
+
+                        if (raw) {
+                            delete raw[key];
+                        } else {
+                            await fs.rm(`${output}${key}.${exportFormat}`);
+                        }
+                        console.log(`Removed "${key}"`);
+                    }
+                    if (raw) {
+                        await fs.writeFile(`${output}${file}`, JSON.stringify(sortKeys({
+                            ...raw,
+                            ...newLipSync ? await getJSON(`${output}${file}`) ?? {} : {} // get new version.
+                        }), null, 4));
+                    }
+
+                    return alreadyLipSynced.length;
+                },
+                list = (await fs.readdir(src)).filter(filter.bind(null, 'mp3')).filter((file) => !check(file.substring(0, file.length - 4)));
+
+            this.differenceOnly = true;
+
+            if (list.length === 0) {
+                // We'll still run the merge in case any have been removed.
+                if (await mergeLipSync()) {
+                    throw Error('Old lip-sync removed, but no new ones required generation.');
+                } else {
+                    throw Error('Lip-sync already up to date.');
+                }
+            }
+            this.updateList = list;
+            config.files = mapReduction(files, list);
+            this.mergeLipSync = mergeLipSync;
+        }
+
+        beforeSend (archive) {
+            const
+                {src} = this.config;
+
+            if (this.differenceOnly) {
+                this.updateList.forEach((file) => archive.file(`${src}${file}`, {
+                    name: file
+                }));
+            } else {
+                archive.directory(src, false);
+            }
+            super.beforeSend(archive);
+        }
+
+        afterExport (...args) {
+            if (this.mergeLipSync) {
+                this.mergeLipSync(true);
+            }
+            super.afterExport(...args)
+        }
+    };
+
+module.exports = LipSync;

package/src/classes/Polly.js

@@ -0,0 +1,10 @@
+const
+    TextToSpeech = require('./TextToSpeech');
+
+module.exports = class Polly extends TextToSpeech {
+    constructor (data) {
+        data.album = data.contents.id;
+        data.encodedBy = 'Amazon Polly';
+        super(data);
+    }
+};

package/src/classes/Rasterize.js

@@ -0,0 +1,13 @@
+const
+    Cheer = require('./Cheer'),
+    Rasterize = class Rasterize extends Cheer {
+        beforeSend (archive) {
+            const
+                {src} = this.config;
+
+            archive.directory(src, false);
+            super.beforeSend(archive);
+        }
+    };
+
+module.exports = Rasterize;

package/src/classes/TextToSpeech.js

@@ -0,0 +1,205 @@
+const
+    Cheer = require('./Cheer'),
+    Transcript = require('./Transcript'),
+    fs = require('fs').promises,
+    getFileData = require('../helpers/getFileData'),
+    id3 = require('node-id3').Promise,
+    isoLanguageMap = {
+        'en-US': 'eng',
+        'es-ES': 'spa'
+    },
+    matches = (file, types) => types.reduce((prev, current) => prev || (file.slice(-(current.length + 1)) === `.${current}`), false),
+    getMP3s = async ({encodedBy = '', output = './', fileTypes = [], transcripts}) => {
+        const
+            all = (await fs.readdir(output))
+                .filter((file) => (file.indexOf('.') !== 0) && (fileTypes.length === 0 || matches(file, fileTypes)));
+
+        if (transcripts) {
+            const
+                keys = Object.keys(transcripts),
+                list = keys.map((id) => `${id}.mp3`),
+                missing = [],
+                present = [];
+
+            // need to check for presence _and_ whether lyrics match.
+            for (let i = 0; i < list.length; i++) {
+                const
+                    file = list[i];
+
+                if (all.indexOf(file) === -1) {
+                    missing.push(file)
+                } else { // not actually missing, so let's check the artist and lyrics.
+                    const
+                        fileHash = (await getFileData(`${output}${file}`, 'mp3')).getHash({
+                            hashKey: 'GenerationHash'
+                        });
+
+                    // We'll assume the original file should be kept if it doesn't include a generation hash.
+                    if (fileHash && fileHash !== transcripts[keys[i]].getHash({
+                        seed: encodedBy
+                    })) {
+                        missing.push(file);
+                    } else {
+                        present.push(file);
+                    }
+                }
+            }
+
+            return {
+                missing,
+                present,
+                unlisted: all.filter((id) => list.indexOf(id) === -1),
+                all
+            };
+        }
+
+        return {
+            all
+        };
+    },
+    captionText = (caption) => (Array.isArray(caption) ? caption : [caption])
+        .map((cap) => cap?.caption ?? cap ?? null)
+        .filter((cap) => cap !== null)
+        .join(' '),
+    clamp = (text, max) => {
+        if (text.length <= max) {
+            return text;
+        } else {
+            const
+                arr = text.substring(0, max).split(' ');
+
+            if (arr.length > 1) {
+                arr.length -= 1; // remove last word which may be partial.
+                return `${arr.join(' ')}...`;
+            } else {
+                return arr[0];
+            }
+        }
+    },
+    appendMP3Meta = ({album, captions, generated = false, language, output, voice}) => {
+        Object.keys(captions).forEach(async (id) => {
+            const
+                text = captionText(captions[id]),
+                tags = {
+                    album,
+                    title: clamp(text, 40),
+                    unsynchronisedLyrics: {
+                        language: isoLanguageMap[language] ?? 'eng',
+                        text
+                    }
+                };
+
+            await id3.update(tags, `${output}${id}.mp3`);
+            
+            if (generated) {
+                console.log(`Voice generated for "${id}".`);
+            } else {
+                console.log(`Voice meta data updated for "${id}".`);
+            }
+        });
+    },
+    mapReduction = (map, list) => list.reduce((obj, file) => {
+        const
+            id = file.substring(0, file.length - 4);
+
+        obj[id] = map[id];
+        return obj;
+    }, {}),
+    TextToSpeech = class extends Cheer {
+        constructor (data) {
+            const
+                {album, encodedBy} = data;
+
+            super(data);
+            this.album = album;
+            this.encodedBy = encodedBy;
+        }
+
+        async prepare (data) {
+            await this.replaceConfigPathWithJSON('script', 'files');
+
+            // create standardized transcripts and an accompanying hash for each.
+            const
+                {files, voice} = this.config,
+                standardizedFiles = {},
+                transcripts = {};
+            
+            Object.keys(files).forEach((key) => {
+                const
+                    transcript = new Transcript(files[key], {voice});
+
+                standardizedFiles[key] = transcript.toJSON();
+                transcripts[key] = transcript;
+            });
+            this.config.files = standardizedFiles;
+            this.transcripts = transcripts;
+
+            return super.prepare(data);
+        }
+
+        async checkDifference () {
+            const
+                {album, config, encodedBy = '', transcripts} = this,
+                {output, files = {}, language, updateAllMetaData = false} = config,
+                {missing, present, unlisted} = await getMP3s({
+                    encodedBy,
+                    fileTypes: ['mp3'],
+                    output,
+                    transcripts
+                });
+    
+            config.files = mapReduction(files, missing);
+    
+            for (let i = 0; i < unlisted.length; i++) {
+                await fs.rm(`${output}${unlisted[i]}`);
+                console.log(`Removed "${unlisted[i]}"`);
+            }
+    
+            if (updateAllMetaData && present.length) {
+                appendMP3Meta({
+                    album,
+                    captions: mapReduction(files, present),
+                    language,
+                    output
+                });
+            }
+    
+            if (missing.length === 0) {
+                throw Error('All voice-over files already exist.');
+            }
+
+            this.updateList = missing;
+        }
+
+        async afterExport (...args) {
+            const
+                {album, encodedBy, config, transcripts, updateList} = this,
+                {files, output, language, voice} = config,
+                list = updateList ?? Object.keys(files).map((id) => `${id}.mp3`);
+
+            appendMP3Meta({
+                album,
+                captions: files,
+                generated: true,
+                language,
+                output,
+                voice
+            });
+            for (let i = 0; i < list.length; i++) {
+                const
+                    filename = list[i],
+                    id = filename.substring(0, filename.length - 4),
+                    file = await getFileData(`${output}${filename}`, 'mp3');
+
+                file.addHash(transcripts[id].getHash({
+                    seed: encodedBy ?? ''
+                }), {
+                    hashKey: 'GenerationHash'
+                });
+                await file.save();
+            }
+            super.afterExport(...args);
+        }
+    };
+
+module.exports = TextToSpeech;

package/src/classes/Transcript.js

@@ -0,0 +1,69 @@
+/**
+ * Processes and manages transcript data with voice resolution and key normalization.
+ * 
+ * This class transforms raw transcript input into a standardized format. It ensures 
+ * every line has an assigned voice, sorts object keys for consistent hashing, 
+ * and provides utility for generating integrity hashes of the content.
+ *
+ * @class Transcript
+ * @param {string|string[]|Object|Object[]} transcript - The raw dialogue or data lines.
+ * @param {Object} options - Configuration object.
+ * @param {string|Object} options.voice - A single voice ID or a mapping of speaker names to IDs.
+ */
+
+const
+    crypto = require('crypto'),
+    sortKeys = (obj) => Object.fromEntries(Object.entries(obj).sort(([a], [b]) => a.localeCompare(b))),
+    resolveVoice = (voice, speaker) => {
+        if (typeof voice === 'string') {
+            return voice;
+        }
+
+        if (speaker && voice[speaker]) {
+            return voice[speaker];
+        }
+
+        return voice.default ?? Object.values(voice)[0];
+    },
+    Transcript = class {
+        constructor (transcript, {voice: voiceOrVoiceMap}) {
+            
+            this.data = (Array.isArray(transcript) ? transcript : [transcript]).map((line) => {
+                if (typeof line === 'string') {
+                    const
+                        result = {caption: line};
+
+                    if (voiceOrVoiceMap) {
+                        result.voice = resolveVoice(voiceOrVoiceMap);
+                    }
+
+                    return result;
+                }
+
+                const
+                    {caption, speaker, voice} = line,
+                    resolvedVoice = voice ?? (voiceOrVoiceMap ? resolveVoice(voiceOrVoiceMap, speaker) : null),
+                    result = {caption};
+
+                if (speaker) {
+                    result.speaker = speaker;
+                }
+
+                if (resolvedVoice) {
+                    result.voice = resolvedVoice;
+                }
+
+                return sortKeys(result);
+            });
+        }
+
+        getHash ({seed = ''} = {}) {
+            return crypto.createHash('sha256').update(seed + JSON.stringify(this.data)).digest('hex');
+        }
+
+        toJSON () {
+            return this.data;
+        }
+    };
+
+module.exports = Transcript;

package/src/classes/Transcription.js

@@ -0,0 +1,163 @@
+const
+    Cheer = require('./Cheer'),
+    Transcript = require('./Transcript'),
+    fs = require('fs').promises,
+    getCaptions = require('../helpers/getCaptions'),
+    getFileData = require('../helpers/getFileData'),
+    getJSON = require('../helpers/getJSON'),
+    SEPARATE_CHARACTER = '|',
+    JOIN_CHARACTER = '^',
+    filter = (fileType, file) => (file.indexOf('.') !== 0) && (file.slice(-(fileType.length + 1)) === `.${fileType}`),
+    mapReduction = (map, list) => list.reduce((obj, file) => {
+        const
+            id = file.substring(0, file.length - 4);
+
+        obj[id] = map[id];
+        return obj;
+    }, {}),
+    Transcription = class Transcription extends Cheer {
+        async prepare (data) {
+            this.differenceOnly = false;
+            this.updateList = null;
+            await this.replaceConfigPathWithJSON('script', 'files');
+            
+            // create standardized transcripts and an accompanying hash for each.
+            const
+                {files, voice} = this.config,
+                standardizedFiles = {},
+                transcripts = {};
+            
+            Object.keys(files).forEach((key) => {
+                const
+                    transcript = new Transcript(files[key], {voice});
+
+                standardizedFiles[key] = transcript.toJSON();
+                transcripts[key] = transcript;
+            });
+            this.config.files = standardizedFiles;
+            this.transcripts = transcripts;
+
+            return super.prepare(data);
+        }
+
+        async checkDifference () {
+            const
+                {config} = this,
+                {files, format = 'json', output, src} = config,
+                fileType = format.toLowerCase(),
+                {captions: alreadyCaptioned, file: captionsFile, files: fileMap} = await getCaptions(output, fileType),
+                check = (id, caption) => {
+                    const
+                        file = fileMap?.[id],
+                        original = alreadyCaptioned[id],
+                        hash = file?.getHash();
+                        
+                    delete alreadyCaptioned[id];
+                    
+                    if (hash) {
+                        const
+                            same = hash === this.transcripts[id].getHash();
+
+                        if (!same) {
+                            console.log(`Updating "${id}".`);
+                        }
+
+                        return same;
+                    } else {
+                        const
+                            cap = (Array.isArray(caption) ? caption : [caption]).map((cap) => cap?.caption ?? cap ?? null).filter((cap) => cap !== null).join(' ').replaceAll(JOIN_CHARACTER, ' ').replaceAll(SEPARATE_CHARACTER, ' ');
+        
+                        if (cap && cap !== original) {
+                            console.log(`Updating "${id}".`);
+                        }
+
+                        // if a caption is not supplied, we'll assume the existence of an original caption means we don't need to redo.
+                        return cap === '' || cap === original;
+                    }
+                },
+                mergeCaptions = async (newCaptions = false) => { // Must run _after_ all checks and will remove what's left.
+                    const
+                        olds = Object.keys(alreadyCaptioned);
+                        
+                    for (let i = 0; i < olds.length; i++) {
+                        const
+                            caption = olds[i];
+    
+                        if (captionsFile) {
+                            captionsFile.removeKey(caption);
+                        } else {
+                            await fs.rm(`${output}${caption}.${fileType}`);
+                        }
+                        console.log(`Removed "${caption}"`);
+                    }
+                    if (captionsFile) {
+                        if (newCaptions) {
+                            captionsFile.mergeData(await getJSON(`${output}captions.json`));
+                        }
+                        await captionsFile.save();
+                    }
+    
+                    return olds.length;
+                },
+                list = (await fs.readdir(src)).filter(filter.bind(null, 'mp3')).filter((file) => {
+                    const
+                        id = file.substring(0, file.length - 4),
+                        caption = files?.[id];
+
+                    return !check(id, caption);
+                });
+
+            this.differenceOnly = true;
+
+            if (list.length === 0) {
+                // We'll still run the merge in case any have been removed.
+                if (await mergeCaptions()) {
+                    throw Error('Old captions removed, but no new captions required generation.');
+                } else {
+                    throw Error('Captions already up to date.');
+                }
+            }
+
+            this.updateList = list;
+            config.files = mapReduction(files, list);
+            this.mergeCaptions = mergeCaptions;
+        }
+
+        beforeSend (archive) {
+            const
+                {src} = this.config;
+
+            if (this.differenceOnly) {
+                this.updateList.forEach((file) => archive.file(`${src}${file}`, {
+                    name: file
+                }));
+            } else {
+                archive.directory(src, false);
+            }
+            super.beforeSend(archive);
+        }
+
+        async afterExport (...args) {
+            const
+                {config, transcripts, updateList} = this,
+                {format = 'json', output} = config;
+
+            if (this.mergeCaptions) {
+                await this.mergeCaptions(true);
+            }
+            if (format !== 'json') {
+                for (let i = 0; i < updateList.length; i++) {
+                    const
+                        filename = updateList[i],
+                        id = filename.substring(0, filename.length - 4),
+                        file = await getFileData(`${output}${id}.${format}`, format);
+
+                    file.addHash(transcripts[id].getHash());
+                    await file.save();
+                }
+            }
+            super.afterExport(...args)
+        }
+    };
+
+module.exports = Transcription;

package/src/formats/File.js

@@ -0,0 +1,27 @@
+const
+    fs = require('fs').promises,
+    File = class {
+        constructor ({fileType}) {
+            this.data = null;
+            this.fileType = fileType;
+        }
+
+        async load (path) {
+            this.path = path;
+            return null;
+        }
+
+        getHash () {
+            return null; // does not support hashes.
+        }
+
+        save () { // return a promise
+            return fs.writeFile(`${this.path}`, this.data ?? '');
+        }
+
+        toString () {
+            return '';
+        }
+    }
+
+module.exports = File;

package/src/formats/JSONFile.js

@@ -0,0 +1,40 @@
+const
+    crypto = require('crypto'),
+    fs = require('fs').promises,
+    File = require("./File"),
+    getJSON = require('../helpers/getJSON'),
+    sortKeys = (obj) => Object.keys(obj).sort().reduce((s, key) => {
+        s[key] = obj[key];
+        return s;
+    }, {});
+
+module.exports = class JSONFile extends File {
+    async load (path) {
+        await super.load(path);
+        this.data = await getJSON(path);
+        return this.data;
+    }
+
+    getHash () {
+        return this.data ? crypto.createHash('sha256').update(JSON.stringify(this.data)).digest('hex') : null;
+    }
+
+    toString () {
+        return Object.keys(this.data).map((key) => this.data[key].map(({content}) => content).join(' ')).join('\n');
+    }
+
+    removeKey (key) {
+        delete this.data[key];
+    }
+
+    mergeData (data) {
+        this.data = sortKeys({
+            ...this.data,
+            ...data
+        });
+    }
+
+    save () { // return a promise
+        return fs.writeFile(`${this.path}`, JSON.stringify(this.data, null, 4));
+    }
+}