@helloao/cli 0.0.4 → 0.0.5

package/db.js

@@ -40,21 +40,17 @@ exports.getPrismaDbFromDir = getPrismaDbFromDir;
 exports.getDbFromDir = getDbFromDir;
 exports.getDb = getDb;
 exports.loadDatasets = loadDatasets;
-exports.serializeFilesForDataset = serializeFilesForDataset;
-exports.serializeFiles = serializeFiles;
+exports.serializeFilesFromDatabase = serializeFilesFromDatabase;
+exports.serializeDatasets = serializeDatasets;
 const prisma_gen_1 = require("./prisma-gen");
 const path_1 = __importStar(require("path"));
 const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
 const fs_extra_1 = require("fs-extra");
 const node_crypto_1 = require("node:crypto");
 const dataset_1 = require("@helloao/tools/generation/dataset");
-const api_1 = require("@helloao/tools/generation/api");
 const files_1 = require("./files");
 const hash_js_1 = require("hash.js");
-const lodash_1 = require("lodash");
-const path_2 = require("path");
-const stream_1 = require("stream");
-const base64_js_1 = require("base64-js");
+const api_1 = require("@helloao/tools/generation/api");
 const cliPath = require.resolve('./index');
 const migrationsPath = path_1.default.resolve((0, path_1.dirname)(cliPath), 'migrations');
 /**
@@ -132,22 +128,22 @@ function getChangedOrNewInputFiles(db, files) {
     });
 }
 function insertFileMetadata(db, files) {
-    const fileUpsert = db.prepare(`INSERT INTO InputFile(
-        translationId,
-        name,
-        format,
-        sha256,
-        sizeInBytes
-    ) VALUES (
-        @translationId,
-        @name,
-        @format,
-        @sha256,
-        @sizeInBytes
-    ) ON CONFLICT(translationId, name) DO 
-        UPDATE SET
-            format=excluded.format,
-            sha256=excluded.sha256,
+    const fileUpsert = db.prepare(`INSERT INTO InputFile(
+        translationId,
+        name,
+        format,
+        sha256,
+        sizeInBytes
+    ) VALUES (
+        @translationId,
+        @name,
+        @format,
+        @sha256,
+        @sizeInBytes
+    ) ON CONFLICT(translationId, name) DO 
+        UPDATE SET
+            format=excluded.format,
+            sha256=excluded.sha256,
             sizeInBytes=excluded.sizeInBytes;`);
     const insertManyFiles = db.transaction((files) => {
         for (let file of files) {
@@ -163,32 +159,32 @@ function insertFileMetadata(db, files) {
     insertManyFiles(files);
 }
 function insertTranslations(db, translations) {
-    const translationUpsert = db.prepare(`INSERT INTO Translation(
-        id,
-        name,
-        language,
-        shortName,
-        textDirection,
-        licenseUrl,
-        website,
-        englishName
-    ) VALUES (
-        @id,
-        @name,
-        @language,
-        @shortName,
-        @textDirection,
-        @licenseUrl,
-        @website,
-        @englishName
-    ) ON CONFLICT(id) DO 
-        UPDATE SET
-            name=excluded.name,
-            language=excluded.language,
-            shortName=excluded.shortName,
-            textDirection=excluded.textDirection,
-            licenseUrl=excluded.licenseUrl,
-            website=excluded.website,
+    const translationUpsert = db.prepare(`INSERT INTO Translation(
+        id,
+        name,
+        language,
+        shortName,
+        textDirection,
+        licenseUrl,
+        website,
+        englishName
+    ) VALUES (
+        @id,
+        @name,
+        @language,
+        @shortName,
+        @textDirection,
+        @licenseUrl,
+        @website,
+        @englishName
+    ) ON CONFLICT(id) DO 
+        UPDATE SET
+            name=excluded.name,
+            language=excluded.language,
+            shortName=excluded.shortName,
+            textDirection=excluded.textDirection,
+            licenseUrl=excluded.licenseUrl,
+            website=excluded.website,
             englishName=excluded.englishName;`);
     const insertManyTranslations = db.transaction((translations) => {
         for (let translation of translations) {
@@ -210,27 +206,27 @@ function insertTranslations(db, translations) {
     }
 }
 function insertTranslationBooks(db, translation, translationBooks) {
-    const bookUpsert = db.prepare(`INSERT INTO Book(
-        id,
-        translationId,
-        title,
-        name,
-        commonName,
-        numberOfChapters,
-        \`order\`
-    ) VALUES (
-        @id,
-        @translationId,
-        @title,
-        @name,
-        @commonName,
-        @numberOfChapters,
-        @bookOrder
-    ) ON CONFLICT(id,translationId) DO 
-        UPDATE SET
-            title=excluded.title,
-            name=excluded.name,
-            commonName=excluded.commonName,
+    const bookUpsert = db.prepare(`INSERT INTO Book(
+        id,
+        translationId,
+        title,
+        name,
+        commonName,
+        numberOfChapters,
+        \`order\`
+    ) VALUES (
+        @id,
+        @translationId,
+        @title,
+        @name,
+        @commonName,
+        @numberOfChapters,
+        @bookOrder
+    ) ON CONFLICT(id,translationId) DO 
+        UPDATE SET
+            title=excluded.title,
+            name=excluded.name,
+            commonName=excluded.commonName,
             numberOfChapters=excluded.numberOfChapters;`);
     const insertMany = db.transaction((books) => {
         for (let book of books) {
@@ -254,69 +250,69 @@ function insertTranslationBooks(db, translation, translationBooks) {
     }
 }
 function insertTranslationContent(db, translation, book, chapters) {
-    const chapterUpsert = db.prepare(`INSERT INTO Chapter(
-        translationId,
-        bookId,
-        number,
-        json
-    ) VALUES (
-        @translationId,
-        @bookId,
-        @number,
-        @json
-    ) ON CONFLICT(translationId,bookId,number) DO 
-        UPDATE SET
+    const chapterUpsert = db.prepare(`INSERT INTO Chapter(
+        translationId,
+        bookId,
+        number,
+        json
+    ) VALUES (
+        @translationId,
+        @bookId,
+        @number,
+        @json
+    ) ON CONFLICT(translationId,bookId,number) DO 
+        UPDATE SET
             json=excluded.json;`);
-    const verseUpsert = db.prepare(`INSERT INTO ChapterVerse(
-        translationId,
-        bookId,
-        chapterNumber,
-        number,
-        text,
-        contentJson
-    ) VALUES (
-        @translationId,
-        @bookId,
-        @chapterNumber,
-        @number,
-        @text,
-        @contentJson
-    ) ON CONFLICT(translationId,bookId,chapterNumber,number) DO 
-        UPDATE SET
-            text=excluded.text,
+    const verseUpsert = db.prepare(`INSERT INTO ChapterVerse(
+        translationId,
+        bookId,
+        chapterNumber,
+        number,
+        text,
+        contentJson
+    ) VALUES (
+        @translationId,
+        @bookId,
+        @chapterNumber,
+        @number,
+        @text,
+        @contentJson
+    ) ON CONFLICT(translationId,bookId,chapterNumber,number) DO 
+        UPDATE SET
+            text=excluded.text,
             contentJson=excluded.contentJson;`);
-    const footnoteUpsert = db.prepare(`INSERT INTO ChapterFootnote(
-        translationId,
-        bookId,
-        chapterNumber,
-        id,
-        verseNumber,
-        text
-    ) VALUES (
-        @translationId,
-        @bookId,
-        @chapterNumber,
-        @id,
-        @verseNumber,
-        @text
-    ) ON CONFLICT(translationId,bookId,chapterNumber,id) DO 
-        UPDATE SET
-            verseNumber=excluded.verseNumber,
+    const footnoteUpsert = db.prepare(`INSERT INTO ChapterFootnote(
+        translationId,
+        bookId,
+        chapterNumber,
+        id,
+        verseNumber,
+        text
+    ) VALUES (
+        @translationId,
+        @bookId,
+        @chapterNumber,
+        @id,
+        @verseNumber,
+        @text
+    ) ON CONFLICT(translationId,bookId,chapterNumber,id) DO 
+        UPDATE SET
+            verseNumber=excluded.verseNumber,
             text=excluded.text;`);
-    const chapterAudioUpsert = db.prepare(`INSERT INTO ChapterAudioUrl(
-        translationId,
-        bookId,
-        number,
-        reader,
-        url
-    ) VALUES (
-        @translationId,
-        @bookId,
-        @number,
-        @reader,
-        @url
-    ) ON CONFLICT(translationId,bookId,number,reader) DO
-        UPDATE SET
+    const chapterAudioUpsert = db.prepare(`INSERT INTO ChapterAudioUrl(
+        translationId,
+        bookId,
+        number,
+        reader,
+        url
+    ) VALUES (
+        @translationId,
+        @bookId,
+        @number,
+        @reader,
+        @url
+    ) ON CONFLICT(translationId,bookId,number,reader) DO
+        UPDATE SET
             url=excluded.url;`);
     const insertChaptersAndVerses = db.transaction(() => {
         for (let chapter of chapters) {
@@ -518,15 +514,15 @@ async function getDbFromDir(dir) {
 }
 async function getDb(dbPath) {
     const db = new better_sqlite3_1.default(dbPath, {});
-    db.exec(`CREATE TABLE IF NOT EXISTS "_prisma_migrations" (
-        "id"                    TEXT PRIMARY KEY NOT NULL,
-        "checksum"              TEXT NOT NULL,
-        "finished_at"           DATETIME,
-        "migration_name"        TEXT NOT NULL,
-        "logs"                  TEXT,
-        "rolled_back_at"        DATETIME,
-        "started_at"            DATETIME NOT NULL DEFAULT current_timestamp,
-        "applied_steps_count"   INTEGER UNSIGNED NOT NULL DEFAULT 0
+    db.exec(`CREATE TABLE IF NOT EXISTS "_prisma_migrations" (
+        "id"                    TEXT PRIMARY KEY NOT NULL,
+        "checksum"              TEXT NOT NULL,
+        "finished_at"           DATETIME,
+        "migration_name"        TEXT NOT NULL,
+        "logs"                  TEXT,
+        "rolled_back_at"        DATETIME,
+        "started_at"            DATETIME NOT NULL DEFAULT current_timestamp,
+        "applied_steps_count"   INTEGER UNSIGNED NOT NULL DEFAULT 0
     );`);
     const migrations = await (0, fs_extra_1.readdir)(migrationsPath);
     const appliedMigrations = db.prepare('SELECT * FROM _prisma_migrations;').all();
@@ -643,112 +639,24 @@ async function* loadDatasets(db, translationsPerBatch = 50, translationsToLoad)
     }
 }
 /**
- * Generates and serializes the API files for the dataset that is stored in the database.
+ * Generates and serializes the API files for the datasets that are stored in the database.
  * Yields each batch of serialized files.
  * @param db The database that the dataset should be loaded from.
- * @param options The options to use for generating the API.
+ * @param options The options to use for serializing the files.
+ * @param apiOptions The options to use for generating the API files.
  * @param translationsPerBatch The number of translations that should be loaded and written per batch.
  * @param translations The list of translations that should be loaded. If not provided, all translations will be loaded.
  */
-async function* serializeFilesForDataset(db, options, translationsPerBatch = 50, translations) {
-    yield* serializeFiles(loadDatasets(db, translationsPerBatch, translations), options);
+function serializeFilesFromDatabase(db, options = {}, translationsPerBatch = 50, translations) {
+    return serializeDatasets(loadDatasets(db, translationsPerBatch, translations), options);
 }
 /**
- * Serializes the API files for the given datasets.
- * @param datasets The dataasets to serialize.
- * @param options The options to use for serializing the files.
+ * Generates and serializes the API files for the given datasets.
+ * Yields each batch of serialized files.
+ *
+ * @param datasets The datasets to serialize.
+ * @param options The options to use for generating and serializing the files.
  */
-async function* serializeFiles(datasets, options) {
-    const mergableFiles = new Map();
-    for await (let dataset of datasets) {
-        const api = (0, api_1.generateApiForDataset)(dataset, options);
-        const files = (0, api_1.generateFilesForApi)(api);
-        console.log('Generated', files.length, 'files');
-        let serializedFiles = [];
-        for (let file of files) {
-            if (file.mergable) {
-                let arr = mergableFiles.get(file.path);
-                if (!arr) {
-                    arr = [];
-                    mergableFiles.set(file.path, arr);
-                }
-                arr.push(file);
-                continue;
-            }
-            const serialized = await transformFile(file.path, file.content);
-            if (serialized) {
-                serializedFiles.push(serialized);
-            }
-        }
-        yield serializedFiles;
-    }
-    let serializedFiles = [];
-    for (let [path, files] of mergableFiles) {
-        let content = {};
-        for (let file of files) {
-            if (!content) {
-                content = file.content;
-            }
-            else {
-                content = (0, lodash_1.mergeWith)(content, file.content, (objValue, srcValue) => {
-                    if (Array.isArray(objValue)) {
-                        return objValue.concat(srcValue);
-                    }
-                    return undefined;
-                });
-            }
-        }
-        if (content) {
-            const serialized = await transformFile(path, content);
-            if (serialized) {
-                serializedFiles.push(serialized);
-            }
-        }
-    }
-    yield serializedFiles;
-    async function transformFile(path, content) {
-        let fileContent;
-        if (typeof content === 'function') {
-            fileContent = await content();
-        }
-        else {
-            fileContent = content;
-        }
-        const ext = (0, path_2.extname)(path);
-        if (ext === '.json') {
-            let json;
-            if (fileContent instanceof ReadableStream) {
-                json = '';
-                for await (const chunk of stream_1.Readable.fromWeb(fileContent, {
-                    encoding: 'utf-8'
-                })) {
-                    json += chunk;
-                }
-            }
-            else {
-                json = JSON.stringify(content, undefined, options.pretty ? 2 : undefined);
-            }
-            return {
-                path,
-                content: json,
-                sha256: () => (0, base64_js_1.fromByteArray)(new Uint8Array((0, hash_js_1.sha256)().update(json).digest()))
-            };
-        }
-        else if (ext === '.mp3') {
-            if (fileContent instanceof ReadableStream) {
-                return {
-                    path,
-                    content: stream_1.Readable.fromWeb(fileContent),
-                };
-            }
-            else {
-                console.warn('Expected content to be a readable stream for', path);
-                console.warn('Skipping file');
-                return null;
-            }
-        }
-        console.warn('Unknown file type', path);
-        console.warn('Skipping file');
-        return null;
-    }
+function serializeDatasets(datasets, options = {}) {
+    return (0, files_1.serializeOutputFiles)((0, api_1.generateOutputFilesFromDatasets)(datasets, options), options);
 }

package/files.d.ts

@@ -1,5 +1,62 @@
-import { InputFile } from "@helloao/tools/generation/common-types";
-import { SerializedFile, Uploader } from "./db";
+import { InputFile, OutputFile } from "@helloao/tools/generation/common-types";
+import { Readable } from "stream";
+/**
+ * Defines an interface that contains information about a serialized file.
+ */
+export interface Uploader {
+    /**
+     * Gets the ideal batch size for the uploader.
+     * Null if the uploader does not need batching.
+     */
+    idealBatchSize: number | null;
+    /**
+     * Uploads the given file.
+     * @param file The file to upload.
+     * @param overwrite Whether the file should be overwritten if it already exists.
+     * @returns True if the file was uploaded. False if the file was skipped due to already existing.
+     */
+    upload(file: SerializedFile, overwrite: boolean): Promise<boolean>;
+    /**
+     * Disposes resources that the uploader uses.
+     */
+    dispose?(): Promise<void>;
+}
+/**
+ * Defines an interface for a file that has been serialized.
+ */
+export interface SerializedFile {
+    path: string;
+    content: string | Readable;
+    /**
+     * Gets the base64-encoded SHA256 hash of the content of the file.
+     */
+    sha256?(): string;
+}
+/**
+ * The options for serializing API files.
+ */
+export interface SerializeApiOptions {
+    /**
+     * Whether the output should be pretty-printed.
+     */
+    pretty?: boolean;
+}
+/**
+ * Serializes the given output files into serialized files using the given options.
+ *
+ * Each iteration of the given files will be processed as a batch, and any mergable files will automatically be merged together and serialized in the final batch.
+ *
+ * @param files The files that should be serialized.
+ * @param options The options for serialization.
+ */
+export declare function serializeOutputFiles(files: AsyncIterable<OutputFile[]>, options: SerializeApiOptions): AsyncGenerator<SerializedFile[]>;
+/**
+ * Serializes the given output file content into a serialized file.
+ * @param path The path that the file should be saved to.
+ * @param content The content of the file.
+ * @param options The options for serialization.
+ */
+export declare function serializeFile(path: string, content: OutputFile['content'], options: SerializeApiOptions): Promise<SerializedFile | null>;
 /**
  * Loads the files for the given translations.
  * @param dir The directory that the translations exist in.

package/files.js

@@ -24,6 +24,8 @@ var __importStar = (this && this.__importStar) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ZipUploader = exports.FilesUploader = void 0;
+exports.serializeOutputFiles = serializeOutputFiles;
+exports.serializeFile = serializeFile;
 exports.loadTranslationsFiles = loadTranslationsFiles;
 exports.loadTranslationFiles = loadTranslationFiles;
 const promises_1 = require("fs/promises");
@@ -34,6 +36,113 @@ const zip_js_1 = require("@zip.js/zip.js");
 const stream_1 = require("stream");
 const hash_js_1 = require("hash.js");
 const usx_parser_1 = require("@helloao/tools/parser/usx-parser");
+const lodash_1 = require("lodash");
+const base64_js_1 = require("base64-js");
+/**
+ * Serializes the given output files into serialized files using the given options.
+ *
+ * Each iteration of the given files will be processed as a batch, and any mergable files will automatically be merged together and serialized in the final batch.
+ *
+ * @param files The files that should be serialized.
+ * @param options The options for serialization.
+ */
+async function* serializeOutputFiles(files, options) {
+    const mergableFiles = new Map();
+    for await (let batch of files) {
+        let serializedFiles = [];
+        for (let file of batch) {
+            if (file.mergable) {
+                let arr = mergableFiles.get(file.path);
+                if (!arr) {
+                    arr = [];
+                    mergableFiles.set(file.path, arr);
+                }
+                arr.push(file);
+                continue;
+            }
+            const serialized = await serializeFile(file.path, file.content, options);
+            if (serialized) {
+                serializedFiles.push(serialized);
+            }
+        }
+        yield serializedFiles;
+    }
+    let serializedFiles = [];
+    for (let [path, files] of mergableFiles) {
+        let content = {};
+        for (let file of files) {
+            if (!content) {
+                content = file.content;
+            }
+            else {
+                content = (0, lodash_1.mergeWith)(content, file.content, (objValue, srcValue) => {
+                    if (Array.isArray(objValue)) {
+                        return objValue.concat(srcValue);
+                    }
+                    return undefined;
+                });
+            }
+        }
+        if (content) {
+            const serialized = await serializeFile(path, content, options);
+            if (serialized) {
+                serializedFiles.push(serialized);
+            }
+        }
+    }
+    yield serializedFiles;
+}
+/**
+ * Serializes the given output file content into a serialized file.
+ * @param path The path that the file should be saved to.
+ * @param content The content of the file.
+ * @param options The options for serialization.
+ */
+async function serializeFile(path, content, options) {
+    let fileContent;
+    if (typeof content === 'function') {
+        fileContent = await content();
+    }
+    else {
+        fileContent = content;
+    }
+    const ext = (0, path_1.extname)(path);
+    if (ext === '.json') {
+        let json;
+        if (fileContent instanceof ReadableStream) {
+            json = '';
+            for await (const chunk of stream_1.Readable.fromWeb(fileContent, {
+                encoding: 'utf-8'
+            })) {
+                json += chunk;
+            }
+        }
+        else {
+            json = JSON.stringify(content, undefined, options.pretty ? 2 : undefined);
+        }
+        return {
+            path,
+            content: json,
+            sha256: () => (0, base64_js_1.fromByteArray)(new Uint8Array((0, hash_js_1.sha256)().update(json).digest()))
+        };
+    }
+    else if (ext === '.mp3') {
+        if (fileContent instanceof ReadableStream) {
+            return {
+                path,
+                content: stream_1.Readable.fromWeb(fileContent),
+            };
+        }
+        else {
+            console.warn('Expected content to be a readable stream for', path);
+            console.warn('Skipping file');
+            return null;
+        }
+    }
+    console.warn('Unknown file type', path);
+    console.warn('Skipping file');
+    return null;
+}
 /**
  * Loads the files for the given translations.
  * @param dir The directory that the translations exist in.