@helloao/cli 0.0.8 → 0.0.9

package/db.js

@@ -1,663 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.importTranslations = importTranslations;
-exports.importTranslationBatch = importTranslationBatch;
-exports.importTranslationFileBatch = importTranslationFileBatch;
-exports.getChangedOrNewInputFiles = getChangedOrNewInputFiles;
-exports.insertFileMetadata = insertFileMetadata;
-exports.insertTranslations = insertTranslations;
-exports.insertTranslationBooks = insertTranslationBooks;
-exports.insertTranslationContent = insertTranslationContent;
-exports.getDbPathFromDir = getDbPathFromDir;
-exports.getDbPath = getDbPath;
-exports.getPrismaDbFromDir = getPrismaDbFromDir;
-exports.getDbFromDir = getDbFromDir;
-exports.getDb = getDb;
-exports.loadDatasets = loadDatasets;
-exports.serializeFilesFromDatabase = serializeFilesFromDatabase;
-exports.serializeDatasets = serializeDatasets;
-const prisma_gen_1 = require("./prisma-gen");
-const path_1 = __importStar(require("path"));
-const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
-const fs_extra_1 = require("fs-extra");
-const node_crypto_1 = require("node:crypto");
-const dataset_1 = require("@helloao/tools/generation/dataset");
-const api_1 = require("@helloao/tools/generation/api");
-const files_1 = require("./files");
-const hash_js_1 = require("hash.js");
-const cliPath = require.resolve('./index');
-const migrationsPath = path_1.default.resolve((0, path_1.dirname)(cliPath), 'migrations');
-/**
- * Imports the translations from the given directories into the database.
- * @param db The database to import the translations into.
- * @param dirs The directories to import the translations from.
- * @param parser The DOM parser that should be used for USX files.
- * @param overwrite Whether to force a reload of the translations.
- */
-async function importTranslations(db, dirs, parser, overwrite) {
-    let batches = [];
-    while (dirs.length > 0) {
-        batches.push(dirs.splice(0, 10));
-    }
-    console.log('Processing', batches.length, 'batches of translations');
-    for (let i = 0; i < batches.length; i++) {
-        const batch = batches[i];
-        console.log(`Processing batch ${i + 1} of ${batches.length}`);
-        await importTranslationBatch(db, batch, parser, overwrite);
-    }
-}
-/**
- * Imports a batch of translations from the given directories into the database.
- * @param db The database to import the translations into.
- * @param dirs The directories that contain the translations.
- * @param parser The DOM parser that should be used for USX files.
- * @param overwrite Whether to force a reload of the translations.
- */
-async function importTranslationBatch(db, dirs, parser, overwrite) {
-    const promises = [];
-    for (let dir of dirs) {
-        const fullPath = path_1.default.resolve(dir);
-        promises.push((0, files_1.loadTranslationFiles)(fullPath).then((files) => files ?? []));
-    }
-    const allFiles = await Promise.all(promises);
-    const files = allFiles.flat();
-    await importTranslationFileBatch(db, files, parser, overwrite);
-}
-/**
- * Parses and imports the given files into the database.
- * @param db The database to import the files into.
- * @param files The files that should be parsed.
- * @param parser The DOM parser that should be used for USX files.
- * @param overwrite Whether to force a reload of the translations.
- */
-async function importTranslationFileBatch(db, files, parser, overwrite) {
-    console.log('Importing', files.length, 'files');
-    if (overwrite) {
-        console.log('Overwriting existing translations.');
-    }
-    const changedFiles = overwrite
-        ? files
-        : getChangedOrNewInputFiles(db, files);
-    console.log('Processing', changedFiles.length, 'changed files');
-    console.log('Skipping', files.length - changedFiles.length, 'unchanged files');
-    const output = (0, dataset_1.generateDataset)(changedFiles, parser);
-    console.log('Generated', output.translations.length, 'translations');
-    insertTranslations(db, output.translations);
-    updateTranslationHashes(db, output.translations);
-    insertFileMetadata(db, changedFiles);
-    console.log(`Inserted ${output.translations} translations into DB`);
-}
-/**
- * Filters the given input files to only include those that have changed.
- * @param db The database to check for changes.
- * @param files The files to filter.
- */
-function getChangedOrNewInputFiles(db, files) {
-    const fileExists = db.prepare('SELECT COUNT(*) as c FROM InputFile WHERE translationId = @translationId AND name = @name AND sha256 = @sha256;');
-    return files.filter((f) => {
-        const count = fileExists.get({
-            translationId: f.metadata.translation.id,
-            name: path_1.default.basename(f.name),
-            sha256: f.sha256,
-        });
-        return count.c <= 0;
-    });
-}
-function insertFileMetadata(db, files) {
-    const fileUpsert = db.prepare(`INSERT INTO InputFile(
-        translationId,
-        name,
-        format,
-        sha256,
-        sizeInBytes
-    ) VALUES (
-        @translationId,
-        @name,
-        @format,
-        @sha256,
-        @sizeInBytes
-    ) ON CONFLICT(translationId, name) DO 
-        UPDATE SET
-            format=excluded.format,
-            sha256=excluded.sha256,
-            sizeInBytes=excluded.sizeInBytes;`);
-    const insertManyFiles = db.transaction((files) => {
-        for (let file of files) {
-            fileUpsert.run({
-                translationId: file.metadata.translation.id,
-                name: path_1.default.basename(file.name),
-                format: file.fileType,
-                sha256: file.sha256,
-                sizeInBytes: file.content.length,
-            });
-        }
-    });
-    insertManyFiles(files);
-}
-function insertTranslations(db, translations) {
-    const translationUpsert = db.prepare(`INSERT INTO Translation(
-        id,
-        name,
-        language,
-        shortName,
-        textDirection,
-        licenseUrl,
-        website,
-        englishName
-    ) VALUES (
-        @id,
-        @name,
-        @language,
-        @shortName,
-        @textDirection,
-        @licenseUrl,
-        @website,
-        @englishName
-    ) ON CONFLICT(id) DO 
-        UPDATE SET
-            name=excluded.name,
-            language=excluded.language,
-            shortName=excluded.shortName,
-            textDirection=excluded.textDirection,
-            licenseUrl=excluded.licenseUrl,
-            website=excluded.website,
-            englishName=excluded.englishName;`);
-    const insertManyTranslations = db.transaction((translations) => {
-        for (let translation of translations) {
-            translationUpsert.run({
-                id: translation.id,
-                name: translation.name,
-                language: translation.language,
-                shortName: translation.shortName,
-                textDirection: translation.textDirection,
-                licenseUrl: translation.licenseUrl,
-                website: translation.website,
-                englishName: translation.englishName,
-            });
-        }
-    });
-    insertManyTranslations(translations);
-    for (let translation of translations) {
-        insertTranslationBooks(db, translation, translation.books);
-    }
-}
-function insertTranslationBooks(db, translation, translationBooks) {
-    const bookUpsert = db.prepare(`INSERT INTO Book(
-        id,
-        translationId,
-        title,
-        name,
-        commonName,
-        numberOfChapters,
-        \`order\`
-    ) VALUES (
-        @id,
-        @translationId,
-        @title,
-        @name,
-        @commonName,
-        @numberOfChapters,
-        @bookOrder
-    ) ON CONFLICT(id,translationId) DO 
-        UPDATE SET
-            title=excluded.title,
-            name=excluded.name,
-            commonName=excluded.commonName,
-            numberOfChapters=excluded.numberOfChapters;`);
-    const insertMany = db.transaction((books) => {
-        for (let book of books) {
-            if (!book) {
-                continue;
-            }
-            bookUpsert.run({
-                id: book.id,
-                translationId: translation.id,
-                title: book.title,
-                name: book.name,
-                commonName: book.commonName,
-                numberOfChapters: book.chapters.length,
-                bookOrder: book.order ?? 9999,
-            });
-        }
-    });
-    insertMany(translationBooks);
-    for (let book of translationBooks) {
-        insertTranslationContent(db, translation, book, book.chapters);
-    }
-}
-function insertTranslationContent(db, translation, book, chapters) {
-    const chapterUpsert = db.prepare(`INSERT INTO Chapter(
-        translationId,
-        bookId,
-        number,
-        json
-    ) VALUES (
-        @translationId,
-        @bookId,
-        @number,
-        @json
-    ) ON CONFLICT(translationId,bookId,number) DO 
-        UPDATE SET
-            json=excluded.json;`);
-    const verseUpsert = db.prepare(`INSERT INTO ChapterVerse(
-        translationId,
-        bookId,
-        chapterNumber,
-        number,
-        text,
-        contentJson
-    ) VALUES (
-        @translationId,
-        @bookId,
-        @chapterNumber,
-        @number,
-        @text,
-        @contentJson
-    ) ON CONFLICT(translationId,bookId,chapterNumber,number) DO 
-        UPDATE SET
-            text=excluded.text,
-            contentJson=excluded.contentJson;`);
-    const footnoteUpsert = db.prepare(`INSERT INTO ChapterFootnote(
-        translationId,
-        bookId,
-        chapterNumber,
-        id,
-        verseNumber,
-        text
-    ) VALUES (
-        @translationId,
-        @bookId,
-        @chapterNumber,
-        @id,
-        @verseNumber,
-        @text
-    ) ON CONFLICT(translationId,bookId,chapterNumber,id) DO 
-        UPDATE SET
-            verseNumber=excluded.verseNumber,
-            text=excluded.text;`);
-    const chapterAudioUpsert = db.prepare(`INSERT INTO ChapterAudioUrl(
-        translationId,
-        bookId,
-        number,
-        reader,
-        url
-    ) VALUES (
-        @translationId,
-        @bookId,
-        @number,
-        @reader,
-        @url
-    ) ON CONFLICT(translationId,bookId,number,reader) DO
-        UPDATE SET
-            url=excluded.url;`);
-    const insertChaptersAndVerses = db.transaction(() => {
-        for (let chapter of chapters) {
-            let verses = [];
-            let footnotes = new Map();
-            for (let c of chapter.chapter.footnotes) {
-                footnotes.set(c.noteId, {
-                    id: c.noteId,
-                    text: c.text,
-                });
-            }
-            for (let c of chapter.chapter.content) {
-                if (c.type === 'verse') {
-                    const verse = c;
-                    if (!verse.number) {
-                        console.error('Verse missing number', translation.id, book.id, chapter.chapter.number, verse.number);
-                        continue;
-                    }
-                    let text = '';
-                    for (let c of verse.content) {
-                        if (typeof c === 'string') {
-                            text += c + ' ';
-                        }
-                        else if (typeof c === 'object') {
-                            if ('lineBreak' in c) {
-                                text += '\n';
-                            }
-                            else if ('text' in c) {
-                                text += c.text + ' ';
-                            }
-                            else if ('noteId' in c) {
-                                const note = footnotes.get(c.noteId);
-                                if (note) {
-                                    note.verseNumber = verse.number;
-                                }
-                            }
-                        }
-                    }
-                    let contentJson = JSON.stringify(verse.content);
-                    verses.push({
-                        number: verse.number,
-                        text: text.trimEnd(),
-                        contentJson,
-                    });
-                }
-            }
-            chapterUpsert.run({
-                translationId: translation.id,
-                bookId: book.id,
-                number: chapter.chapter.number,
-                json: JSON.stringify(chapter.chapter),
-            });
-            for (let verse of verses) {
-                verseUpsert.run({
-                    translationId: translation.id,
-                    bookId: book.id,
-                    chapterNumber: chapter.chapter.number,
-                    number: verse.number,
-                    text: verse.text,
-                    contentJson: verse.contentJson,
-                });
-            }
-            for (let footnote of footnotes.values()) {
-                footnoteUpsert.run({
-                    translationId: translation.id,
-                    bookId: book.id,
-                    chapterNumber: chapter.chapter.number,
-                    id: footnote.id,
-                    verseNumber: footnote.verseNumber,
-                    text: footnote.text,
-                });
-            }
-            for (let reader in chapter.thisChapterAudioLinks) {
-                const url = chapter.thisChapterAudioLinks[reader];
-                if (url) {
-                    chapterAudioUpsert.run({
-                        translationId: translation.id,
-                        bookId: book.id,
-                        number: chapter.chapter.number,
-                        reader: reader,
-                        url,
-                    });
-                }
-            }
-        }
-    });
-    insertChaptersAndVerses();
-}
-/**
- * Updates the hashes for the translations in the database.
- * @param db The database to update the hashes in.
- * @param translations The translations to update the hashes for.
- */
-function updateTranslationHashes(db, translations) {
-    console.log(`Updating hashes for ${translations.length} translations.`);
-    const updateTranslationHash = db.prepare(`UPDATE Translation SET sha256 = @sha256 WHERE id = @translationId;`);
-    const updateBookHash = db.prepare(`UPDATE Book SET sha256 = @sha256 WHERE translationId = @translationId AND id = @bookId;`);
-    const updateChapterHash = db.prepare(`UPDATE Chapter SET sha256 = @sha256 WHERE translationId = @translationId AND bookId = @bookId AND number = @chapterNumber;`);
-    const getBooks = db.prepare('SELECT * FROM Book WHERE translationId = ?;');
-    const getChapters = db.prepare('SELECT * FROM Chapter WHERE translationId = @translationId AND bookId = @bookId;');
-    for (let translation of translations) {
-        const translationSha = (0, hash_js_1.sha256)()
-            .update(translation.id)
-            .update(translation.name)
-            .update(translation.language)
-            .update(translation.licenseUrl)
-            .update(translation.textDirection)
-            .update(translation.website)
-            .update(translation.englishName)
-            .update(translation.shortName);
-        const books = getBooks.all(translation.id);
-        for (let book of books) {
-            const chapters = getChapters.all({
-                translationId: translation.id,
-                bookId: book.id,
-            });
-            const bookSha = (0, hash_js_1.sha256)()
-                .update(book.translationId)
-                .update(book.id)
-                .update(book.numberOfChapters)
-                .update(book.order)
-                .update(book.name)
-                .update(book.title)
-                .update(book.commonName);
-            for (let chapter of chapters) {
-                const hash = (0, hash_js_1.sha256)()
-                    .update(chapter.translationId)
-                    .update(chapter.bookId)
-                    .update(chapter.number)
-                    .update(chapter.json)
-                    .digest('hex');
-                chapter.sha256 = hash;
-                bookSha.update(hash);
-            }
-            const updateChapters = db.transaction(() => {
-                for (let chapter of chapters) {
-                    updateChapterHash.run({
-                        sha256: chapter.sha256,
-                        translationId: chapter.translationId,
-                        bookId: chapter.bookId,
-                        chapterNumber: chapter.number,
-                    });
-                }
-            });
-            updateChapters();
-            const bookHash = bookSha.digest('hex');
-            book.sha256 = bookHash;
-            translationSha.update(bookHash);
-        }
-        const updateBooks = db.transaction(() => {
-            for (let book of books) {
-                updateBookHash.run({
-                    sha256: book.sha256,
-                    translationId: book.translationId,
-                    bookId: book.id,
-                });
-            }
-        });
-        updateBooks();
-        const hash = translationSha.digest('hex');
-        translation.sha256 = hash;
-    }
-    const updateTranslations = db.transaction(() => {
-        for (let translation of translations) {
-            updateTranslationHash.run({
-                sha256: translation.sha256,
-                translationId: translation.id,
-            });
-        }
-    });
-    updateTranslations();
-    console.log(`Updated.`);
-}
-function getDbPathFromDir(dir) {
-    dir = dir || process.cwd();
-    return path_1.default.resolve(dir, 'bible-api.db');
-}
-function getDbPath(p) {
-    if (p) {
-        return path_1.default.resolve(p);
-    }
-    return getDbPathFromDir(process.cwd());
-}
-function getPrismaDbFromDir(dir) {
-    const dbPath = getDbPathFromDir(dir);
-    const prisma = new prisma_gen_1.PrismaClient({
-        datasources: {
-            db: {
-                url: `file:${dbPath}`,
-            },
-        },
-    });
-    return prisma;
-}
-async function getDbFromDir(dir) {
-    const dbPath = getDbPathFromDir(dir);
-    const db = await getDb(dbPath);
-    return db;
-}
-async function getDb(dbPath) {
-    const db = new better_sqlite3_1.default(dbPath, {});
-    db.exec(`CREATE TABLE IF NOT EXISTS "_prisma_migrations" (
-        "id"                    TEXT PRIMARY KEY NOT NULL,
-        "checksum"              TEXT NOT NULL,
-        "finished_at"           DATETIME,
-        "migration_name"        TEXT NOT NULL,
-        "logs"                  TEXT,
-        "rolled_back_at"        DATETIME,
-        "started_at"            DATETIME NOT NULL DEFAULT current_timestamp,
-        "applied_steps_count"   INTEGER UNSIGNED NOT NULL DEFAULT 0
-    );`);
-    const migrations = await (0, fs_extra_1.readdir)(migrationsPath);
-    const appliedMigrations = db
-        .prepare('SELECT * FROM _prisma_migrations;')
-        .all();
-    let missingMigrations = [];
-    for (let migration of migrations) {
-        if (appliedMigrations.some((m) => m.migration_name === migration)) {
-            continue;
-        }
-        if (path_1.default.extname(migration) !== '') {
-            continue;
-        }
-        missingMigrations.push(migration);
-    }
-    const insertMigrationStatement = db.prepare('INSERT INTO _prisma_migrations (id, checksum, started_at, finished_at, migration_name, applied_steps_count, logs, rolled_back_at) VALUES (?, ?, ?, ?, ?, ?, NULL, NULL);');
-    for (let missingMigration of missingMigrations) {
-        console.log(`Applying migration ${missingMigration}...`);
-        const migration = path_1.default.resolve(migrationsPath, missingMigration, 'migration.sql');
-        const migrationFile = await (0, fs_extra_1.readFile)(migration, 'utf8');
-        db.exec(migrationFile);
-        insertMigrationStatement.run((0, node_crypto_1.randomUUID)(), '', new Date().toISOString(), new Date().toISOString(), missingMigration, 1);
-    }
-    return db;
-}
-/**
- * Loads the datasets from the database in a series of batches.
- * @param db The database.
- * @param translationsPerBatch The number of translations to load per batch.
- * @param translationsToLoad The list of translations to load. If not provided, all translations will be loaded.
- */
-async function* loadDatasets(db, translationsPerBatch = 50, translationsToLoad) {
-    let offset = 0;
-    let pageSize = translationsPerBatch;
-    console.log('Generating API files in batches of', pageSize);
-    const totalTranslations = await db.translation.count();
-    const totalBatches = Math.ceil(totalTranslations / pageSize);
-    let batchNumber = 1;
-    while (true) {
-        console.log('Generating API batch', batchNumber, 'of', totalBatches);
-        batchNumber++;
-        const query = {
-            skip: offset,
-            take: pageSize,
-        };
-        if (translationsToLoad && translationsToLoad.length > 0) {
-            query.where = {
-                id: {
-                    in: translationsToLoad,
-                },
-            };
-        }
-        const translations = await db.translation.findMany(query);
-        if (translations.length <= 0) {
-            break;
-        }
-        const dataset = {
-            translations: [],
-        };
-        for (let translation of translations) {
-            const datasetTranslation = {
-                ...translation,
-                shortName: translation.shortName,
-                textDirection: translation.textDirection,
-                books: [],
-            };
-            dataset.translations.push(datasetTranslation);
-            const books = await db.book.findMany({
-                where: {
-                    translationId: translation.id,
-                },
-                orderBy: {
-                    order: 'asc',
-                },
-            });
-            for (let book of books) {
-                const chapters = await db.chapter.findMany({
-                    where: {
-                        translationId: translation.id,
-                        bookId: book.id,
-                    },
-                    orderBy: {
-                        number: 'asc',
-                    },
-                });
-                const audioLinks = await db.chapterAudioUrl.findMany({
-                    where: {
-                        translationId: translation.id,
-                        bookId: book.id,
-                    },
-                    orderBy: [{ number: 'asc' }, { reader: 'asc' }],
-                });
-                const bookChapters = chapters.map((chapter) => {
-                    return {
-                        chapter: JSON.parse(chapter.json),
-                        thisChapterAudioLinks: audioLinks
-                            .filter((link) => link.number === chapter.number)
-                            .reduce((acc, link) => {
-                            acc[link.reader] = link.url;
-                            return acc;
-                        }, {}),
-                    };
-                });
-                const datasetBook = {
-                    ...book,
-                    chapters: bookChapters,
-                };
-                datasetTranslation.books.push(datasetBook);
-            }
-        }
-        yield dataset;
-        offset += pageSize;
-    }
-}
-/**
- * Generates and serializes the API files for the datasets that are stored in the database.
- * Yields each batch of serialized files.
- * @param db The database that the dataset should be loaded from.
- * @param options The options to use for serializing the files.
- * @param apiOptions The options to use for generating the API files.
- * @param translationsPerBatch The number of translations that should be loaded and written per batch.
- * @param translations The list of translations that should be loaded. If not provided, all translations will be loaded.
- */
-function serializeFilesFromDatabase(db, options = {}, translationsPerBatch = 50, translations) {
-    return serializeDatasets(loadDatasets(db, translationsPerBatch, translations), options);
-}
-/**
- * Generates and serializes the API files for the given datasets.
- * Yields each batch of serialized files.
- *
- * @param datasets The datasets to serialize.
- * @param options The options to use for generating and serializing the files.
- */
-function serializeDatasets(datasets, options = {}) {
-    return (0, files_1.serializeOutputFiles)((0, api_1.generateOutputFilesFromDatasets)(datasets, options), options);
-}

package/downloads.js

@@ -1,12 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.downloadFile = downloadFile;
-const fs_extra_1 = require("fs-extra");
-const node_stream_1 = require("node:stream");
-const promises_1 = require("node:stream/promises");
-async function downloadFile(url, path) {
-    console.log('Downloading', url, 'to', path);
-    const reader = await fetch(url).then(r => r.body);
-    const writeStream = (0, fs_extra_1.createWriteStream)(path);
-    await (0, promises_1.finished)(node_stream_1.Readable.fromWeb(reader).pipe(writeStream));
-}