@helloao/cli 0.0.1

package/actions.d.ts

@@ -0,0 +1,57 @@
+export interface InitDbOptions {
+    /**
+     * The path to the source database to copy the schema from.
+     */
+    source?: string;
+    /**
+     * The languages to copy from the source database. If not specified, then all languages will be copied.
+     */
+    language?: string[];
+}
+/**
+ * Initializes a new Bible API DB.
+ * @param dbPath The path to the database. If null or empty, then the "bible-api.db" will be used from the current working directory.
+ * @param options The options for the initialization.
+ */
+export declare function initDb(dbPath: string | null, options: InitDbOptions): Promise<void>;
+export interface ImportTranslationOptions {
+    /**
+     * Whether to forcibly import the translations, even if they have already been imported.
+     */
+    overwrite?: boolean;
+}
+/**
+ * Imports a translation from the given directory into the database in the current working directory.
+ * @param dir The directory that the translation is located in.
+ * @param dirs Any extra directories that should be imported.
+ * @param options The options for the import.
+ */
+export declare function importTranslation(dir: string, dirs: string[], options: ImportTranslationOptions): Promise<void>;
+/**
+ * Imports all the translations from the given directory into the database in the current working directory.
+ * @param dir The directory that the translations are located in.
+ * @param options The options.
+ */
+export declare function importTranslations(dir: string, options: ImportTranslationOptions): Promise<void>;
+export interface FetchTranslationsOptions {
+    /**
+     * Fetch all translations. If omitted, only undownloaded translations will be fetched.
+     */
+    all?: boolean;
+}
+/**
+ * Fetches the specified translations from fetch.bible and places them in the given directory.
+ * @param dir The directory that the translations should be placed in.
+ * @param translations The translations that should be downloaded. If not specified, then all translations will be downloaded.
+ * @param options The options.
+ */
+export declare function fetchTranslations(dir: string, translations?: string[], options?: FetchTranslationsOptions): Promise<void>;
+/**
+ * Fetches the specified audio translations and places them in the given directory.
+ * Translations should be in the format "translationId/audioId". e.g. "BSB/gilbert"
+ * @param dir The directory that the translations should be placed in.
+ * @param translations The translations that should be downloaded.
+ * @param options The options.
+ */
+export declare function fetchAudio(dir: string, translations: string[], options?: FetchTranslationsOptions): Promise<void>;
+//# sourceMappingURL=actions.d.ts.map

package/actions.js

@@ -0,0 +1,262 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.initDb = initDb;
+exports.importTranslation = importTranslation;
+exports.importTranslations = importTranslations;
+exports.fetchTranslations = fetchTranslations;
+exports.fetchAudio = fetchAudio;
+const node_path_1 = __importStar(require("node:path"));
+const database = __importStar(require("./db"));
+const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
+const linkedom_1 = require("linkedom");
+const promises_1 = require("node:fs/promises");
+const fetch_client_1 = require("@gracious.tech/fetch-client");
+const utils_1 = require("@helloao/tools/utils");
+const fs_extra_1 = require("fs-extra");
+const audio_1 = require("@helloao/tools/generation/audio");
+const book_order_1 = require("@helloao/tools/generation/book-order");
+const downloads_1 = require("./downloads");
+/**
+ * Initializes a new Bible API DB.
+ * @param dbPath The path to the database. If null or empty, then the "bible-api.db" will be used from the current working directory.
+ * @param options The options for the initialization.
+ */
+async function initDb(dbPath, options) {
+    console.log('Initializing new Bible API DB...');
+    if (options.source) {
+        const db = new better_sqlite3_1.default(database.getDbPath(dbPath), {});
+        const sourcePath = node_path_1.default.resolve(options.source);
+        try {
+            console.log('Copying schema from source DB...');
+            if (options.language) {
+                console.log('Copying only the following languages:', options.language);
+                const languages = `(${options.language.map((l) => `'${l}'`).join(', ')})`;
+                db.exec(`
+                    ATTACH DATABASE "${sourcePath}" AS source;
+
+                    CREATE TABLE "_prisma_migrations" AS SELECT * FROM source._prisma_migrations;
+                    
+                    CREATE TABLE "Translation" AS SELECT * FROM source.Translation
+                    WHERE language IN ${languages};
+
+                    CREATE TABLE "Book" AS SELECT * FROM source.Book
+                    INNER JOIN source.Translation ON source.Translation.id = source.Book.translationId
+                    WHERE source.Translation.language IN ${languages};
+
+                    CREATE TABLE "Chapter" AS SELECT * FROM source.Chapter
+                    INNER JOIN source.Translation ON source.Translation.id = source.Chapter.translationId
+                    WHERE source.Translation.language IN ${languages};
+
+                    CREATE TABLE "ChapterVerse" AS SELECT * FROM source.ChapterVerse
+                    INNER JOIN source.Translation ON source.Translation.id = source.ChapterVerse.translationId
+                    WHERE source.Translation.language IN ${languages};
+
+                    CREATE TABLE "ChapterFootnote" AS SELECT * FROM source.ChapterFootnote
+                    INNER JOIN source.Translation ON source.Translation.id = source.ChapterFootnote.translationId
+                    WHERE source.Translation.language IN ${languages};
+
+                    CREATE TABLE "ChapterAudioUrl" AS SELECT * FROM source.ChapterAudioUrl
+                    INNER JOIN source.Translation ON source.Translation.id = source.ChapterAudioUrl.translationId
+                    WHERE source.Translation.language IN ${languages};
+                `);
+            }
+            else {
+                db.exec(`
+                    ATTACH DATABASE "${sourcePath}" AS source;
+
+                    CREATE TABLE "_prisma_migrations" AS SELECT * FROM source._prisma_migrations;
+                    CREATE TABLE "Translation" AS SELECT * FROM source.Translation;
+                    CREATE TABLE "Book" AS SELECT * FROM source.Book;
+                    CREATE TABLE "Chapter" AS SELECT * FROM source.Chapter;
+                    CREATE TABLE "ChapterVerse" AS SELECT * FROM source.ChapterVerse;
+                    CREATE TABLE "ChapterFootnote" AS SELECT * FROM source.ChapterFootnote;
+                    CREATE TABLE "ChapterAudioUrl" AS SELECT * FROM source.ChapterAudioUrl;
+                `);
+            }
+            console.log('Done.');
+        }
+        finally {
+            db.close();
+        }
+    }
+    else {
+        const db = await database.getDb(database.getDbPath(dbPath));
+        db.close();
+    }
+}
+/**
+ * Imports a translation from the given directory into the database in the current working directory.
+ * @param dir The directory that the translation is located in.
+ * @param dirs Any extra directories that should be imported.
+ * @param options The options for the import.
+ */
+async function importTranslation(dir, dirs, options) {
+    const parser = new linkedom_1.DOMParser();
+    globalThis.DOMParser = linkedom_1.DOMParser;
+    globalThis.Element = linkedom_1.Element;
+    globalThis.Node = linkedom_1.Node;
+    const db = await database.getDbFromDir(process.cwd());
+    try {
+        await database.importTranslations(db, [dir, ...dirs], parser, !!options.overwrite);
+    }
+    finally {
+        db.close();
+    }
+}
+/**
+ * Imports all the translations from the given directory into the database in the current working directory.
+ * @param dir The directory that the translations are located in.
+ * @param options The options.
+ */
+async function importTranslations(dir, options) {
+    const parser = new linkedom_1.DOMParser();
+    globalThis.DOMParser = linkedom_1.DOMParser;
+    globalThis.Element = linkedom_1.Element;
+    globalThis.Node = linkedom_1.Node;
+    const db = await database.getDbFromDir(process.cwd());
+    try {
+        const files = await (0, promises_1.readdir)(dir);
+        const translationDirs = files.map(f => node_path_1.default.resolve(dir, f));
+        console.log(`Importing ${translationDirs.length} translations`);
+        await database.importTranslations(db, translationDirs, parser, !!options.overwrite);
+    }
+    finally {
+        db.close();
+    }
+}
+/**
+ * Fetches the specified translations from fetch.bible and places them in the given directory.
+ * @param dir The directory that the translations should be placed in.
+ * @param translations The translations that should be downloaded. If not specified, then all translations will be downloaded.
+ * @param options The options.
+ */
+async function fetchTranslations(dir, translations, options = {}) {
+    const translationsSet = new Set(translations);
+    const client = new fetch_client_1.BibleClient({
+        remember_fetches: false,
+    });
+    const collection = await client.fetch_collection();
+    const collectionTranslations = collection.get_translations();
+    console.log(`Discovered ${collectionTranslations.length} translations`);
+    const filtered = translations && translations.length <= 0 ? collectionTranslations : collectionTranslations.filter(t => translationsSet.has(t.id));
+    let batches = [];
+    while (filtered.length > 0) {
+        batches.push(filtered.splice(0, 10));
+    }
+    console.log(`Downloading ${filtered.length} translations in ${batches.length} batches`);
+    for (let i = 0; i < batches.length; i++) {
+        const batch = batches[i];
+        console.log(`Downloading batch ${i + 1} of ${batches.length}`);
+        const translations = await Promise.all(batch.map(async (t) => {
+            const id = (0, utils_1.getTranslationId)(t.id);
+            const translation = {
+                id,
+                name: (0, utils_1.getFirstNonEmpty)(t.name_local, t.name_english, t.name_abbrev),
+                direction: (0, utils_1.getFirstNonEmpty)(t.direction, 'ltr'),
+                englishName: (0, utils_1.getFirstNonEmpty)(t.name_english, t.name_abbrev, t.name_local),
+                language: (0, utils_1.normalizeLanguage)(t.language),
+                licenseUrl: t.attribution_url,
+                shortName: (0, utils_1.getFirstNonEmpty)(t.name_abbrev, id),
+                website: t.attribution_url,
+            };
+            const books = await Promise.all(collection.get_books(t.id).map(async (b) => {
+                const name = `${b.id}.usx`;
+                if (!options.all && await (0, fs_extra_1.exists)(node_path_1.default.resolve(dir, translation.id, name))) {
+                    return null;
+                }
+                const content = await collection.fetch_book(t.id, b.id, 'usx');
+                const contentString = content.get_whole();
+                const file = {
+                    fileType: 'usx',
+                    content: contentString,
+                    metadata: {
+                        translation
+                    },
+                    name,
+                };
+                return file;
+            }));
+            return {
+                translation,
+                books,
+            };
+        }));
+        console.log(`Writing batch ${i + 1} of ${batches.length}`);
+        let promises = [];
+        for (let { translation, books } of translations) {
+            for (let book of books) {
+                if (!book) {
+                    continue;
+                }
+                if (!book.name) {
+                    throw new Error('Book name is required');
+                }
+                const fullPath = node_path_1.default.resolve(dir, translation.id, book.name);
+                await (0, promises_1.mkdir)(node_path_1.default.dirname(fullPath), { recursive: true });
+                const promise = (0, promises_1.writeFile)(fullPath, book.content);
+                promises.push(promise);
+            }
+            const translationPath = node_path_1.default.resolve(dir, translation.id, 'metadata.json');
+            await (0, promises_1.mkdir)(node_path_1.default.dirname(translationPath), { recursive: true });
+            const translationData = JSON.stringify(translation, null, 2);
+            promises.push((0, promises_1.writeFile)(translationPath, translationData));
+        }
+        await Promise.all(promises);
+    }
+}
+/**
+ * Fetches the specified audio translations and places them in the given directory.
+ * Translations should be in the format "translationId/audioId". e.g. "BSB/gilbert"
+ * @param dir The directory that the translations should be placed in.
+ * @param translations The translations that should be downloaded.
+ * @param options The options.
+ */
+async function fetchAudio(dir, translations, options = {}) {
+    for (let translation of translations) {
+        const [translationId, reader] = translation.split('/');
+        const generator = audio_1.KNOWN_AUDIO_TRANSLATIONS.get(translationId)?.get(reader);
+        if (!generator) {
+            console.warn('Unknown translation:', translation);
+            continue;
+        }
+        for (let [bookId, chapters] of book_order_1.bookChapterCountMap) {
+            for (let chapter = 1; chapter <= chapters; chapter++) {
+                const url = generator(bookId, chapter);
+                const ext = (0, node_path_1.extname)(url);
+                const [translationId, reader] = translation.split('/');
+                const name = `${chapter}.${reader}${ext}`;
+                const fullPath = node_path_1.default.resolve(dir, 'audio', translationId, bookId, name);
+                if (!options.all && await (0, fs_extra_1.exists)(fullPath)) {
+                    continue;
+                }
+                await (0, downloads_1.downloadFile)(url, fullPath);
+            }
+        }
+    }
+}

package/cli.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=cli.d.ts.map

package/cli.js

@@ -0,0 +1,139 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const commander_1 = require("commander");
+const path_1 = __importDefault(require("path"));
+const promises_1 = require("fs/promises");
+const linkedom_1 = require("linkedom");
+const downloads_1 = require("./downloads");
+const uploads_1 = require("./uploads");
+const actions_1 = require("./actions");
+const files_1 = require("./files");
+const dataset_1 = require("@helloao/tools/generation/dataset");
+const iterators_1 = require("@helloao/tools/parser/iterators");
+async function start() {
+    const parser = new linkedom_1.DOMParser();
+    globalThis.DOMParser = linkedom_1.DOMParser;
+    globalThis.Element = linkedom_1.Element;
+    globalThis.Node = linkedom_1.Node;
+    const program = new commander_1.Command();
+    program.name('bible-api')
+        .description('A CLI for managing a Bible API.')
+        .version('0.0.1');
+    program.command('init [path]')
+        .description('Initialize a new Bible API DB.')
+        .option('--source <path>', 'The source database to copy from.')
+        .option('--language <languages...>', 'The language(s) that the database should be initialized with.')
+        .action(async (dbPath, options) => {
+        await (0, actions_1.initDb)(dbPath, options);
+    });
+    program.command('import-translation <dir> [dirs...]')
+        .description('Imports a translation from the given directory into the database.')
+        .option('--overwrite', 'Whether to overwrite existing files.')
+        .action(async (dir, dirs, options) => {
+        await (0, actions_1.importTranslation)(dir, dirs, options);
+    });
+    program.command('import-translations <dir>')
+        .description('Imports all translations from the given directory into the database.')
+        .option('--overwrite', 'Whether to overwrite existing files.')
+        .action(async (dir, options) => {
+        await (0, actions_1.importTranslations)(dir, options);
+    });
+    program.command('generate-translation-files <input> <dir>')
+        .description('Generates API files from the given input translation.')
+        .option('--batch-size <size>', 'The number of translations to generate API files for in each batch.', '50')
+        .option('--translations <translations...>', 'The translations to generate API files for.')
+        .option('--overwrite', 'Whether to overwrite existing files.')
+        .option('--overwrite-common-files', 'Whether to overwrite only common files.')
+        .option('--file-pattern <pattern>', 'The file pattern regex that should be used to filter the files that are generated.')
+        .option('--use-common-name', 'Whether to use the common name for the book chapter API link. If false, then book IDs are used.')
+        .option('--generate-audio-files', 'Whether to replace the audio URLs in the dataset with ones that are hosted locally.')
+        .option('--profile <profile>', 'The AWS profile to use for uploading to S3.')
+        .option('--pretty', 'Whether to generate pretty-printed JSON files.')
+        .action(async (input, dest, options) => {
+        const parser = new linkedom_1.DOMParser();
+        globalThis.DOMParser = linkedom_1.DOMParser;
+        globalThis.Element = linkedom_1.Element;
+        globalThis.Node = linkedom_1.Node;
+        const files = await (0, files_1.loadTranslationFiles)(path_1.default.resolve(input));
+        const dataset = (0, dataset_1.generateDataset)(files, parser);
+        await (0, uploads_1.uploadApiFiles)(path_1.default.resolve(dest), options, (0, iterators_1.toAsyncIterable)([dataset]));
+    });
+    program.command('generate-translations-files <input> <dir>')
+        .description('Generates API files from the given input translations.')
+        .option('--batch-size <size>', 'The number of translations to generate API files for in each batch.', '50')
+        .option('--translations <translations...>', 'The translations to generate API files for.')
+        .option('--overwrite', 'Whether to overwrite existing files.')
+        .option('--overwrite-common-files', 'Whether to overwrite only common files.')
+        .option('--file-pattern <pattern>', 'The file pattern regex that should be used to filter the files that are uploaded.')
+        .option('--use-common-name', 'Whether to use the common name for the book chapter API link. If false, then book IDs are used.')
+        .option('--generate-audio-files', 'Whether to replace the audio URLs in the dataset with ones that are hosted locally.')
+        .option('--profile <profile>', 'The AWS profile to use for uploading to S3.')
+        .option('--pretty', 'Whether to generate pretty-printed JSON files.')
+        .action(async (input, dest, options) => {
+        const parser = new linkedom_1.DOMParser();
+        globalThis.DOMParser = linkedom_1.DOMParser;
+        globalThis.Element = linkedom_1.Element;
+        globalThis.Node = linkedom_1.Node;
+        const dirs = await (0, promises_1.readdir)(path_1.default.resolve(input));
+        const batchSize = parseInt(options.batchSize);
+        for (let b of (0, iterators_1.batch)(dirs, batchSize)) {
+            const files = await (0, files_1.loadTranslationsFiles)(b);
+            const dataset = (0, dataset_1.generateDataset)(files, parser);
+            await (0, uploads_1.uploadApiFiles)(dest, options, (0, iterators_1.toAsyncIterable)([dataset]));
+        }
+    });
+    program.command('upload-api-files')
+        .argument('<dest>', 'The destination to upload the API files to.')
+        .description('Uploads API files to the specified destination. For S3, use the format s3://bucket-name/path/to/folder.')
+        .option('--batch-size <size>', 'The number of translations to generate API files for in each batch.', '50')
+        .option('--translations <translations...>', 'The translations to generate API files for.')
+        .option('--overwrite', 'Whether to overwrite existing files.')
+        .option('--overwrite-common-files', 'Whether to overwrite only common files.')
+        .option('--file-pattern <pattern>', 'The file pattern regex that should be used to filter the files that are uploaded.')
+        .option('--use-common-name', 'Whether to use the common name for the book chapter API link. If false, then book IDs are used.')
+        .option('--generate-audio-files', 'Whether to replace the audio URLs in the dataset with ones that are hosted locally.')
+        .option('--profile <profile>', 'The AWS profile to use for uploading to S3.')
+        .option('--pretty', 'Whether to generate pretty-printed JSON files.')
+        .action(async (dest, options) => {
+        await (0, uploads_1.uploadApiFilesFromDatabase)(dest, options);
+    });
+    program.command('fetch-translations <dir> [translations...]')
+        .description('Fetches the specified translations from fetch.bible and places them in the given directory.')
+        .option('-a, --all', 'Fetch all translations. If omitted, only undownloaded translations will be fetched.')
+        .action(async (dir, translations, options) => {
+        await (0, actions_1.fetchTranslations)(dir, translations, options);
+    });
+    program.command('fetch-audio <dir> [translations...]')
+        .description('Fetches the specified audio translations and places them in the given directory.\nTranslations should be in the format "translationId/audioId". e.g. "BSB/gilbert"')
+        .option('-a, --all', 'Fetch all translations. If omitted, only undownloaded translations will be fetched.')
+        .action(async (dir, translations, options) => {
+        await (0, actions_1.fetchAudio)(dir, translations, options);
+    });
+    program.command('fetch-bible-metadata <dir>')
+        .description('Fetches the Theographic bible metadata and places it in the given directory.')
+        .action(async (dir) => {
+        let files = [
+            'books.json',
+            'chapters.json',
+            'easton.json',
+            'events.json',
+            'people.json',
+            'peopleGroups.json',
+            'periods.json',
+            'places.json',
+            'verses.json',
+        ];
+        await (0, promises_1.mkdir)(dir, { recursive: true });
+        let promises = files.map(async (file) => {
+            const url = `https://raw.githubusercontent.com/robertrouse/theographic-bible-metadata/master/json/${file}`;
+            const fullPath = path_1.default.resolve(dir, file);
+            await (0, downloads_1.downloadFile)(url, fullPath);
+        });
+        await Promise.all(promises);
+    });
+    await program.parseAsync(process.argv);
+}
+start();

package/db.d.ts

@@ -0,0 +1,110 @@
+import { PrismaClient } from "./prisma-gen";
+import { Database } from 'better-sqlite3';
+import { DatasetOutput, DatasetTranslation, DatasetTranslationBook } from "@helloao/tools/generation/dataset";
+import { InputFile, TranslationBookChapter } from "@helloao/tools/generation";
+import { GenerateApiOptions } from "@helloao/tools/generation/api";
+import { DOMParser } from "linkedom";
+import { Readable } from "stream";
+/**
+ * Imports the translations from the given directories into the database.
+ * @param db The database to import the translations into.
+ * @param dirs The directories to import the translations from.
+ * @param parser The DOM parser that should be used for USX files.
+ * @param overwrite Whether to force a reload of the translations.
+ */
+export declare function importTranslations(db: Database, dirs: string[], parser: DOMParser, overwrite: boolean): Promise<void>;
+/**
+ * Imports a batch of translations from the given directories into the database.
+ * @param db The database to import the translations into.
+ * @param dirs The directories that contain the translations.
+ * @param parser The DOM parser that should be used for USX files.
+ * @param overwrite Whether to force a reload of the translations.
+ */
+export declare function importTranslationBatch(db: Database, dirs: string[], parser: DOMParser, overwrite: boolean): Promise<void>;
+/**
+ * Parses and imports the given files into the database.
+ * @param db The database to import the files into.
+ * @param files The files that should be parsed.
+ * @param parser The DOM parser that should be used for USX files.
+ * @param overwrite Whether to force a reload of the translations.
+ */
+export declare function importTranslationFileBatch(db: Database, files: InputFile[], parser: DOMParser, overwrite: boolean): Promise<void>;
+/**
+ * Filters the given input files to only include those that have changed.
+ * @param db The database to check for changes.
+ * @param files The files to filter.
+ */
+export declare function getChangedOrNewInputFiles(db: Database, files: InputFile[]): InputFile[];
+export declare function insertFileMetadata(db: Database, files: InputFile[]): void;
+export declare function insertTranslations(db: Database, translations: DatasetTranslation[]): void;
+export declare function insertTranslationBooks(db: Database, translation: DatasetTranslation, translationBooks: DatasetTranslationBook[]): void;
+export declare function insertTranslationContent(db: Database, translation: DatasetTranslation, book: DatasetTranslationBook, chapters: TranslationBookChapter[]): void;
+export declare function getDbPathFromDir(dir: string): string;
+export declare function getDbPath(p: string | null): string;
+export declare function getPrismaDbFromDir(dir: string): PrismaClient<{
+    datasources: {
+        db: {
+            url: string;
+        };
+    };
+}, never, import("prisma-gen/runtime/library").DefaultArgs>;
+export declare function getDbFromDir(dir: string): Promise<Database>;
+export declare function getDb(dbPath: string): Promise<Database>;
+export interface SerializedFile {
+    path: string;
+    content: string | Readable;
+    /**
+     * Gets the base64-encoded SHA256 hash of the content of the file.
+     */
+    sha256?(): string;
+}
+/**
+ * Loads the datasets from the database in a series of batches.
+ * @param db The database.
+ * @param translationsPerBatch The number of translations to load per batch.
+ * @param translationsToLoad The list of translations to load. If not provided, all translations will be loaded.
+ */
+export declare function loadDatasets(db: PrismaClient, translationsPerBatch?: number, translationsToLoad?: string[]): AsyncGenerator<DatasetOutput>;
+export interface SerializeApiOptions extends GenerateApiOptions {
+    /**
+     * Whether the output should be pretty-printed.
+     */
+    pretty?: boolean;
+}
+/**
+ * Generates and serializes the API files for the dataset that is stored in the database.
+ * Yields each batch of serialized files.
+ * @param db The database that the dataset should be loaded from.
+ * @param options The options to use for generating the API.
+ * @param translationsPerBatch The number of translations that should be loaded and written per batch.
+ * @param translations The list of translations that should be loaded. If not provided, all translations will be loaded.
+ */
+export declare function serializeFilesForDataset(db: PrismaClient, options: SerializeApiOptions, translationsPerBatch?: number, translations?: string[]): AsyncGenerator<SerializedFile[]>;
+/**
+ * Serializes the API files for the given datasets.
+ * @param datasets The dataasets to serialize.
+ * @param options The options to use for serializing the files.
+ */
+export declare function serializeFiles(datasets: AsyncIterable<DatasetOutput>, options: SerializeApiOptions): AsyncGenerator<SerializedFile[]>;
+/**
+ * Defines an interface that contains information about a serialized file.
+ */
+export interface Uploader {
+    /**
+     * Gets the ideal batch size for the uploader.
+     * Null if the uploader does not need batching.
+     */
+    idealBatchSize: number | null;
+    /**
+     * Uploads the given file.
+     * @param file The file to upload.
+     * @param overwrite Whether the file should be overwritten if it already exists.
+     * @returns True if the file was uploaded. False if the file was skipped due to already existing.
+     */
+    upload(file: SerializedFile, overwrite: boolean): Promise<boolean>;
+    /**
+     * Disposes resources that the uploader uses.
+     */
+    dispose?(): Promise<void>;
+}
+//# sourceMappingURL=db.d.ts.map