@helloao/cli 0.0.5 → 0.0.6

package/db.js

@@ -48,9 +48,9 @@ const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
 const fs_extra_1 = require("fs-extra");
 const node_crypto_1 = require("node:crypto");
 const dataset_1 = require("@helloao/tools/generation/dataset");
+const api_1 = require("@helloao/tools/generation/api");
 const files_1 = require("./files");
 const hash_js_1 = require("hash.js");
-const api_1 = require("@helloao/tools/generation/api");
 const cliPath = require.resolve('./index');
 const migrationsPath = path_1.default.resolve((0, path_1.dirname)(cliPath), 'migrations');
 /**
@@ -101,7 +101,9 @@ async function importTranslationFileBatch(db, files, parser, overwrite) {
     if (overwrite) {
         console.log('Overwriting existing translations.');
     }
-    const changedFiles = overwrite ? files : getChangedOrNewInputFiles(db, files);
+    const changedFiles = overwrite
+        ? files
+        : getChangedOrNewInputFiles(db, files);
     console.log('Processing', changedFiles.length, 'changed files');
     console.log('Skipping', files.length - changedFiles.length, 'unchanged files');
     const output = (0, dataset_1.generateDataset)(changedFiles, parser);
@@ -118,7 +120,7 @@ async function importTranslationFileBatch(db, files, parser, overwrite) {
  */
 function getChangedOrNewInputFiles(db, files) {
     const fileExists = db.prepare('SELECT COUNT(*) as c FROM InputFile WHERE translationId = @translationId AND name = @name AND sha256 = @sha256;');
-    return files.filter(f => {
+    return files.filter((f) => {
         const count = fileExists.get({
             translationId: f.metadata.translation.id,
             name: path_1.default.basename(f.name),
@@ -128,22 +130,22 @@ function getChangedOrNewInputFiles(db, files) {
     });
 }
 function insertFileMetadata(db, files) {
-    const fileUpsert = db.prepare(`INSERT INTO InputFile(
-        translationId,
-        name,
-        format,
-        sha256,
-        sizeInBytes
-    ) VALUES (
-        @translationId,
-        @name,
-        @format,
-        @sha256,
-        @sizeInBytes
-    ) ON CONFLICT(translationId, name) DO 
-        UPDATE SET
-            format=excluded.format,
-            sha256=excluded.sha256,
+    const fileUpsert = db.prepare(`INSERT INTO InputFile(
+        translationId,
+        name,
+        format,
+        sha256,
+        sizeInBytes
+    ) VALUES (
+        @translationId,
+        @name,
+        @format,
+        @sha256,
+        @sizeInBytes
+    ) ON CONFLICT(translationId, name) DO 
+        UPDATE SET
+            format=excluded.format,
+            sha256=excluded.sha256,
             sizeInBytes=excluded.sizeInBytes;`);
     const insertManyFiles = db.transaction((files) => {
         for (let file of files) {
@@ -159,32 +161,32 @@ function insertFileMetadata(db, files) {
     insertManyFiles(files);
 }
 function insertTranslations(db, translations) {
-    const translationUpsert = db.prepare(`INSERT INTO Translation(
-        id,
-        name,
-        language,
-        shortName,
-        textDirection,
-        licenseUrl,
-        website,
-        englishName
-    ) VALUES (
-        @id,
-        @name,
-        @language,
-        @shortName,
-        @textDirection,
-        @licenseUrl,
-        @website,
-        @englishName
-    ) ON CONFLICT(id) DO 
-        UPDATE SET
-            name=excluded.name,
-            language=excluded.language,
-            shortName=excluded.shortName,
-            textDirection=excluded.textDirection,
-            licenseUrl=excluded.licenseUrl,
-            website=excluded.website,
+    const translationUpsert = db.prepare(`INSERT INTO Translation(
+        id,
+        name,
+        language,
+        shortName,
+        textDirection,
+        licenseUrl,
+        website,
+        englishName
+    ) VALUES (
+        @id,
+        @name,
+        @language,
+        @shortName,
+        @textDirection,
+        @licenseUrl,
+        @website,
+        @englishName
+    ) ON CONFLICT(id) DO 
+        UPDATE SET
+            name=excluded.name,
+            language=excluded.language,
+            shortName=excluded.shortName,
+            textDirection=excluded.textDirection,
+            licenseUrl=excluded.licenseUrl,
+            website=excluded.website,
             englishName=excluded.englishName;`);
     const insertManyTranslations = db.transaction((translations) => {
         for (let translation of translations) {
@@ -206,27 +208,27 @@ function insertTranslations(db, translations) {
     }
 }
 function insertTranslationBooks(db, translation, translationBooks) {
-    const bookUpsert = db.prepare(`INSERT INTO Book(
-        id,
-        translationId,
-        title,
-        name,
-        commonName,
-        numberOfChapters,
-        \`order\`
-    ) VALUES (
-        @id,
-        @translationId,
-        @title,
-        @name,
-        @commonName,
-        @numberOfChapters,
-        @bookOrder
-    ) ON CONFLICT(id,translationId) DO 
-        UPDATE SET
-            title=excluded.title,
-            name=excluded.name,
-            commonName=excluded.commonName,
+    const bookUpsert = db.prepare(`INSERT INTO Book(
+        id,
+        translationId,
+        title,
+        name,
+        commonName,
+        numberOfChapters,
+        \`order\`
+    ) VALUES (
+        @id,
+        @translationId,
+        @title,
+        @name,
+        @commonName,
+        @numberOfChapters,
+        @bookOrder
+    ) ON CONFLICT(id,translationId) DO 
+        UPDATE SET
+            title=excluded.title,
+            name=excluded.name,
+            commonName=excluded.commonName,
             numberOfChapters=excluded.numberOfChapters;`);
     const insertMany = db.transaction((books) => {
         for (let book of books) {
@@ -240,7 +242,7 @@ function insertTranslationBooks(db, translation, translationBooks) {
                 name: book.name,
                 commonName: book.commonName,
                 numberOfChapters: book.chapters.length,
-                bookOrder: book.order ?? 9999
+                bookOrder: book.order ?? 9999,
             });
         }
     });
@@ -250,69 +252,69 @@ function insertTranslationBooks(db, translation, translationBooks) {
     }
 }
 function insertTranslationContent(db, translation, book, chapters) {
-    const chapterUpsert = db.prepare(`INSERT INTO Chapter(
-        translationId,
-        bookId,
-        number,
-        json
-    ) VALUES (
-        @translationId,
-        @bookId,
-        @number,
-        @json
-    ) ON CONFLICT(translationId,bookId,number) DO 
-        UPDATE SET
+    const chapterUpsert = db.prepare(`INSERT INTO Chapter(
+        translationId,
+        bookId,
+        number,
+        json
+    ) VALUES (
+        @translationId,
+        @bookId,
+        @number,
+        @json
+    ) ON CONFLICT(translationId,bookId,number) DO 
+        UPDATE SET
             json=excluded.json;`);
-    const verseUpsert = db.prepare(`INSERT INTO ChapterVerse(
-        translationId,
-        bookId,
-        chapterNumber,
-        number,
-        text,
-        contentJson
-    ) VALUES (
-        @translationId,
-        @bookId,
-        @chapterNumber,
-        @number,
-        @text,
-        @contentJson
-    ) ON CONFLICT(translationId,bookId,chapterNumber,number) DO 
-        UPDATE SET
-            text=excluded.text,
+    const verseUpsert = db.prepare(`INSERT INTO ChapterVerse(
+        translationId,
+        bookId,
+        chapterNumber,
+        number,
+        text,
+        contentJson
+    ) VALUES (
+        @translationId,
+        @bookId,
+        @chapterNumber,
+        @number,
+        @text,
+        @contentJson
+    ) ON CONFLICT(translationId,bookId,chapterNumber,number) DO 
+        UPDATE SET
+            text=excluded.text,
             contentJson=excluded.contentJson;`);
-    const footnoteUpsert = db.prepare(`INSERT INTO ChapterFootnote(
-        translationId,
-        bookId,
-        chapterNumber,
-        id,
-        verseNumber,
-        text
-    ) VALUES (
-        @translationId,
-        @bookId,
-        @chapterNumber,
-        @id,
-        @verseNumber,
-        @text
-    ) ON CONFLICT(translationId,bookId,chapterNumber,id) DO 
-        UPDATE SET
-            verseNumber=excluded.verseNumber,
+    const footnoteUpsert = db.prepare(`INSERT INTO ChapterFootnote(
+        translationId,
+        bookId,
+        chapterNumber,
+        id,
+        verseNumber,
+        text
+    ) VALUES (
+        @translationId,
+        @bookId,
+        @chapterNumber,
+        @id,
+        @verseNumber,
+        @text
+    ) ON CONFLICT(translationId,bookId,chapterNumber,id) DO 
+        UPDATE SET
+            verseNumber=excluded.verseNumber,
             text=excluded.text;`);
-    const chapterAudioUpsert = db.prepare(`INSERT INTO ChapterAudioUrl(
-        translationId,
-        bookId,
-        number,
-        reader,
-        url
-    ) VALUES (
-        @translationId,
-        @bookId,
-        @number,
-        @reader,
-        @url
-    ) ON CONFLICT(translationId,bookId,number,reader) DO
-        UPDATE SET
+    const chapterAudioUpsert = db.prepare(`INSERT INTO ChapterAudioUrl(
+        translationId,
+        bookId,
+        number,
+        reader,
+        url
+    ) VALUES (
+        @translationId,
+        @bookId,
+        @number,
+        @reader,
+        @url
+    ) ON CONFLICT(translationId,bookId,number,reader) DO
+        UPDATE SET
             url=excluded.url;`);
     const insertChaptersAndVerses = db.transaction(() => {
         for (let chapter of chapters) {
@@ -427,7 +429,7 @@ function updateTranslationHashes(db, translations) {
         for (let book of books) {
             const chapters = getChapters.all({
                 translationId: translation.id,
-                bookId: book.id
+                bookId: book.id,
             });
             const bookSha = (0, hash_js_1.sha256)()
                 .update(book.translationId)
@@ -453,7 +455,7 @@ function updateTranslationHashes(db, translations) {
                         sha256: chapter.sha256,
                         translationId: chapter.translationId,
                         bookId: chapter.bookId,
-                        chapterNumber: chapter.number
+                        chapterNumber: chapter.number,
                     });
                 }
             });
@@ -467,7 +469,7 @@ function updateTranslationHashes(db, translations) {
                 updateBookHash.run({
                     sha256: book.sha256,
                     translationId: book.translationId,
-                    bookId: book.id
+                    bookId: book.id,
                 });
             }
         });
@@ -479,7 +481,7 @@ function updateTranslationHashes(db, translations) {
         for (let translation of translations) {
             updateTranslationHash.run({
                 sha256: translation.sha256,
-                translationId: translation.id
+                translationId: translation.id,
             });
         }
     });
@@ -502,8 +504,8 @@ function getPrismaDbFromDir(dir) {
         datasources: {
             db: {
                 url: `file:${dbPath}`,
-            }
-        }
+            },
+        },
     });
     return prisma;
 }
@@ -514,21 +516,23 @@ async function getDbFromDir(dir) {
 }
 async function getDb(dbPath) {
     const db = new better_sqlite3_1.default(dbPath, {});
-    db.exec(`CREATE TABLE IF NOT EXISTS "_prisma_migrations" (
-        "id"                    TEXT PRIMARY KEY NOT NULL,
-        "checksum"              TEXT NOT NULL,
-        "finished_at"           DATETIME,
-        "migration_name"        TEXT NOT NULL,
-        "logs"                  TEXT,
-        "rolled_back_at"        DATETIME,
-        "started_at"            DATETIME NOT NULL DEFAULT current_timestamp,
-        "applied_steps_count"   INTEGER UNSIGNED NOT NULL DEFAULT 0
+    db.exec(`CREATE TABLE IF NOT EXISTS "_prisma_migrations" (
+        "id"                    TEXT PRIMARY KEY NOT NULL,
+        "checksum"              TEXT NOT NULL,
+        "finished_at"           DATETIME,
+        "migration_name"        TEXT NOT NULL,
+        "logs"                  TEXT,
+        "rolled_back_at"        DATETIME,
+        "started_at"            DATETIME NOT NULL DEFAULT current_timestamp,
+        "applied_steps_count"   INTEGER UNSIGNED NOT NULL DEFAULT 0
     );`);
     const migrations = await (0, fs_extra_1.readdir)(migrationsPath);
-    const appliedMigrations = db.prepare('SELECT * FROM _prisma_migrations;').all();
+    const appliedMigrations = db
+        .prepare('SELECT * FROM _prisma_migrations;')
+        .all();
     let missingMigrations = [];
     for (let migration of migrations) {
-        if (appliedMigrations.some(m => m.migration_name === migration)) {
+        if (appliedMigrations.some((m) => m.migration_name === migration)) {
             continue;
         }
         if (path_1.default.extname(migration) !== '') {
@@ -570,7 +574,7 @@ async function* loadDatasets(db, translationsPerBatch = 50, translationsToLoad)
             query.where = {
                 id: {
                     in: translationsToLoad,
-                }
+                },
             };
         }
         const translations = await db.translation.findMany(query);
@@ -578,7 +582,7 @@ async function* loadDatasets(db, translationsPerBatch = 50, translationsToLoad)
             break;
         }
         const dataset = {
-            translations: []
+            translations: [],
         };
         for (let translation of translations) {
             const datasetTranslation = {
@@ -609,22 +613,19 @@ async function* loadDatasets(db, translationsPerBatch = 50, translationsToLoad)
                 const audioLinks = await db.chapterAudioUrl.findMany({
                     where: {
                         translationId: translation.id,
-                        bookId: book.id
+                        bookId: book.id,
                     },
-                    orderBy: [
-                        { number: 'asc' },
-                        { reader: 'asc' }
-                    ]
+                    orderBy: [{ number: 'asc' }, { reader: 'asc' }],
                 });
-                const bookChapters = chapters.map(chapter => {
+                const bookChapters = chapters.map((chapter) => {
                     return {
                         chapter: JSON.parse(chapter.json),
                         thisChapterAudioLinks: audioLinks
-                            .filter(link => link.number === chapter.number)
+                            .filter((link) => link.number === chapter.number)
                             .reduce((acc, link) => {
                             acc[link.reader] = link.url;
                             return acc;
-                        }, {})
+                        }, {}),
                     };
                 });
                 const datasetBook = {

package/files.d.ts

@@ -1,5 +1,5 @@
-import { InputFile, OutputFile } from "@helloao/tools/generation/common-types";
-import { Readable } from "stream";
+import { InputFile, OutputFile } from '@helloao/tools/generation/common-types';
+import { Readable } from 'stream';
 /**
  * Defines an interface that contains information about a serialized file.
  */
@@ -110,4 +110,9 @@ export declare class ZipUploader implements Uploader {
     upload(file: SerializedFile, _overwrite: boolean): Promise<boolean>;
     dispose(): Promise<void>;
 }
+/**
+ * Calculates the SHa256 hash of the given input files.
+ * @param files The files to hash.
+ */
+export declare function hashInputFiles(files: InputFile[]): string;
 //# sourceMappingURL=files.d.ts.map

package/files.js

@@ -28,6 +28,7 @@ exports.serializeOutputFiles = serializeOutputFiles;
 exports.serializeFile = serializeFile;
 exports.loadTranslationsFiles = loadTranslationsFiles;
 exports.loadTranslationFiles = loadTranslationFiles;
+exports.hashInputFiles = hashInputFiles;
 const promises_1 = require("fs/promises");
 const path_1 = require("path");
 const path = __importStar(require("path"));
@@ -112,7 +113,7 @@ async function serializeFile(path, content, options) {
         if (fileContent instanceof ReadableStream) {
             json = '';
             for await (const chunk of stream_1.Readable.fromWeb(fileContent, {
-                encoding: 'utf-8'
+                encoding: 'utf-8',
             })) {
                 json += chunk;
             }
@@ -123,7 +124,7 @@ async function serializeFile(path, content, options) {
         return {
             path,
             content: json,
-            sha256: () => (0, base64_js_1.fromByteArray)(new Uint8Array((0, hash_js_1.sha256)().update(json).digest()))
+            sha256: () => (0, base64_js_1.fromByteArray)(new Uint8Array((0, hash_js_1.sha256)().update(json).digest())),
         };
     }
     else if (ext === '.mp3') {
@@ -169,12 +170,14 @@ async function loadTranslationFiles(translation) {
         return [];
     }
     let files = await (0, promises_1.readdir)(translation);
-    let usfmFiles = files.filter(f => (0, path_1.extname)(f) === '.usfm' || (0, path_1.extname)(f) === '.usx' || (0, path_1.extname)(f) === '.json');
+    let usfmFiles = files.filter((f) => (0, path_1.extname)(f) === '.usfm' ||
+        (0, path_1.extname)(f) === '.usx' ||
+        (0, path_1.extname)(f) === '.json');
     if (usfmFiles.length <= 0) {
         translation = path.resolve(translation, 'usfm');
         if ((0, fs_extra_1.existsSync)(translation)) {
             files = await (0, promises_1.readdir)(translation);
-            usfmFiles = files.filter(f => (0, path_1.extname)(f) === '.usfm');
+            usfmFiles = files.filter((f) => (0, path_1.extname)(f) === '.usfm');
         }
     }
     if (usfmFiles.length <= 0) {
@@ -188,7 +191,7 @@ async function loadTranslationFiles(translation) {
         }
         const filePath = path.resolve(translation, file);
         promises.push(loadFile(filePath, {
-            translation: metadata
+            translation: metadata,
         }));
     }
     return await Promise.all(promises);
@@ -216,13 +219,15 @@ async function loadTranslationMetadata(translation) {
                 licenseUrl: metadata.copyright.attribution_url,
                 website: metadata.copyright.attribution_url,
                 shortName: metadata.name.abbrev,
-                direction: metadata.direction
+                direction: metadata.direction,
             };
         }
         else {
             const metadataJson = path.resolve(translation, 'metadata.json');
             if ((0, fs_extra_1.existsSync)(metadataJson)) {
-                const data = await (0, promises_1.readFile)(metadataJson, { encoding: 'utf-8' });
+                const data = await (0, promises_1.readFile)(metadataJson, {
+                    encoding: 'utf-8',
+                });
                 return JSON.parse(data);
             }
         }
@@ -238,7 +243,7 @@ async function loadTranslationMetadata(translation) {
 async function loadFile(file, metadata) {
     const extension = path.extname(file);
     const content = await (0, promises_1.readFile)(file, {
-        encoding: 'utf-8'
+        encoding: 'utf-8',
     });
     const hash = (0, hash_js_1.sha256)()
         .update(content)
@@ -339,3 +344,19 @@ function makeRelativePath(path) {
     }
     return path;
 }
+/**
+ * Calculates the SHa256 hash of the given input files.
+ * @param files The files to hash.
+ */
+function hashInputFiles(files) {
+    let sha = (0, hash_js_1.sha256)();
+    for (let file of files) {
+        if (file.sha256) {
+            sha.update(file.sha256);
+        }
+        else {
+            sha.update(file.content);
+        }
+    }
+    return sha.digest('hex');
+}

package/migrations/20240623183848_add_book_order/migration.sql

@@ -1,26 +1,26 @@
-/*
-  Warnings:
-
-  - Added the required column `order` to the `Book` table without a default value. This is not possible if the table is not empty.
-
-*/
--- RedefineTables
-PRAGMA defer_foreign_keys=ON;
-PRAGMA foreign_keys=OFF;
-CREATE TABLE "new_Book" (
-    "id" TEXT NOT NULL,
-    "translationId" TEXT NOT NULL,
-    "name" TEXT NOT NULL,
-    "commonName" TEXT NOT NULL,
-    "title" TEXT,
-    "order" INTEGER NOT NULL,
-    "numberOfChapters" INTEGER NOT NULL,
-
-    PRIMARY KEY ("translationId", "id"),
-    CONSTRAINT "Book_translationId_fkey" FOREIGN KEY ("translationId") REFERENCES "Translation" ("id") ON DELETE RESTRICT ON UPDATE CASCADE
-);
-INSERT INTO "new_Book" ("commonName", "id", "name", "numberOfChapters", "title", "translationId") SELECT "commonName", "id", "name", "numberOfChapters", "title", "translationId" FROM "Book";
-DROP TABLE "Book";
-ALTER TABLE "new_Book" RENAME TO "Book";
-PRAGMA foreign_keys=ON;
-PRAGMA defer_foreign_keys=OFF;
+/*
+  Warnings:
+
+  - Added the required column `order` to the `Book` table without a default value. This is not possible if the table is not empty.
+
+*/
+-- RedefineTables
+PRAGMA defer_foreign_keys=ON;
+PRAGMA foreign_keys=OFF;
+CREATE TABLE "new_Book" (
+    "id" TEXT NOT NULL,
+    "translationId" TEXT NOT NULL,
+    "name" TEXT NOT NULL,
+    "commonName" TEXT NOT NULL,
+    "title" TEXT,
+    "order" INTEGER NOT NULL,
+    "numberOfChapters" INTEGER NOT NULL,
+
+    PRIMARY KEY ("translationId", "id"),
+    CONSTRAINT "Book_translationId_fkey" FOREIGN KEY ("translationId") REFERENCES "Translation" ("id") ON DELETE RESTRICT ON UPDATE CASCADE
+);
+INSERT INTO "new_Book" ("commonName", "id", "name", "numberOfChapters", "title", "translationId") SELECT "commonName", "id", "name", "numberOfChapters", "title", "translationId" FROM "Book";
+DROP TABLE "Book";
+ALTER TABLE "new_Book" RENAME TO "Book";
+PRAGMA foreign_keys=ON;
+PRAGMA defer_foreign_keys=OFF;