@helloao/cli 0.0.4 → 0.0.6

package/README.md

@@ -34,4 +34,92 @@ Commands:
                                                         Translations should be in the format "translationId/audioId". e.g. "BSB/gilbert"
   fetch-bible-metadata <dir>                            Fetches the Theographic bible metadata and places it in the given directory.
   help [command]                                        display help for command
-```
+```
+
+The `@helloao/cli` package can also be used as a library.
+
+The library exports a variety of actions, utilities, and supporting classes designed to assist with generating and managing a Free Use Bible API.
+
+There are 6 main exports:
+
+-   `actions` - This export contains function versions of the CLI commands. They make it easy to call a CLI command from a script.
+-   `db` - This export contains functions that make working with a database easier. It supports operations like importing translations into a database, inserting chapters, verses, etc. and getting an updated database instance from a path.
+-   `downloads` - This export contains functions that make downloading files easier.
+-   `files` - This export contains functions that make working with files easier. It has functions to load files from a translation, discover translation metadata from the filesystem, and classes that support uploading API files to the local file system or to a zip archive.
+-   `uploads` - This export contains functions that make it easy to upload an API to a destination like S3, the local filesystem, or a zip archive.
+-   `s3` - This export contains a class that can upload files to S3.
+
+Here are some common operations that you might want to perform:
+
+#### Get a SQL Database
+
+```typescript
+import { db } from '@helloao/cli';
+
+const pathToDb = './bible-database.db';
+const database = await db.getDb(pathToDb);
+
+// do work on the database
+
+// Close it when you are done.
+database.close();
+```
+
+#### Import a translation into a database from a directory
+
+```typescript
+import { db } from '@helloao/cli';
+
+const pathToDb = './bible-database.db';
+const database = await db.getDb(pathToDb);
+
+// Get a DOMParser for parsing USX.
+// On Node.js, you may have to import jsdom or linkedom.
+const parser = new DOMParser();
+
+const pathToTranslation = './path/to/translation';
+
+// Whether to overwrite files that already exist in the database.
+// The system will automatically determine the hashes of the input files and overwrite changed files if needed, so this is only needed
+// when you know that they need to be overwritten.
+const overwrite = false;
+await db.importTranslations(database, pathToTranslation, parser, overwrite);
+```
+
+#### Generate an API from a translation
+
+```typescript
+import { files, uploads } from '@helloao/cli';
+import { generation } from '@helloao/tools';
+import { toAsyncIterable } from '@helloao/tools/parser/iterators';
+
+const translationPath = './path/to/translation';
+const translationFiles = await files.loadTranslationFiles(translationPath);
+
+// Used to parse XML
+const domParser = new DOMParser();
+
+// Generate a dataset from the files
+// Datasets organize all the files and their content
+// by translation, book, chapter, and verse
+const dataset = generation.dataset.generateDataset(files, parser);
+
+// You can optionally specifiy a prefix that should be added to all API
+// links
+const pathPrefix = '';
+
+// Generate an API representation from the files
+// This adds links between chapters and additional metadata.
+const api = generation.api.generateApiForDataset(dataset, {
+    pathPrefix,
+});
+
+// Generate output files from the API representation.
+// This will give us a list of files and file paths that represent
+// the entire API.
+const outputFiles = generation.api.generateFilesForApi(api);
+
+// Optionally upload files by using:
+// const dest = 's3://my-bucket';
+// await uploads.serializeAndUploadDatasets(dest, toAsyncIterable(outputFiles));
+```

package/actions.d.ts

@@ -1,3 +1,4 @@
+import { UploadApiFromDatabaseOptions, UploadApiOptions } from './uploads';
 export interface InitDbOptions {
     /**
      * The path to the source database to copy the schema from.
@@ -54,4 +55,67 @@ export declare function fetchTranslations(dir: string, translations?: string[],
  * @param options The options.
  */
 export declare function fetchAudio(dir: string, translations: string[], options?: FetchTranslationsOptions): Promise<void>;
+/**
+ * Generates the translation files directly from the translations stored in the given input directory.
+ * @param input The input directory that the translations are stored in.
+ * @param dest The destination to upload the API files to.
+ * @param options The options for the generation.
+ */
+export declare function generateTranslationsFiles(input: string, dest: string, options: UploadApiFromDatabaseOptions): Promise<void>;
+/**
+ * Generates the translation files directly from the translation stored in the given input directory.
+ * @param input The input directory that the translation is stored in.
+ * @param dest The destination to upload the API files to.
+ * @param options The options for the generation.
+ */
+export declare function generateTranslationFiles(input: string, dest: string, options: UploadApiOptions): Promise<void>;
+/**
+ * The options for uploading the test translations.
+ */
+export interface UploadTestTranslationOptions extends UploadApiOptions {
+    /**
+     * The s3 URL to upload the translations to.
+     * Defaults to "s3://ao-bible-api-public-uploads"
+     */
+    s3Url?: string;
+}
+export interface UploadTestTranslationResult {
+    /**
+     * The S3 URL where the translations were uploaded to.
+     */
+    uploadS3Url: string;
+    /**
+     * The HTTP URL that the version can be accessed at.
+     */
+    url: string;
+    /**
+     * The URL that the available translations can be accessed at.
+     */
+    availableTranslationsUrl: string;
+    /**
+     * The version that was uploaded.
+     * This is a SHA-256 hash of the input files.
+     */
+    version: string;
+}
+/**
+ * Generates the API files directly from the translations stored in the given input directory and
+ * uploads them to the HelloAO test s3 bucket.
+ *
+ * Requires access to the HelloAO test s3 bucket. Email hello@helloao.org for access.
+ *
+ * @param input The input directory that the translations are stored in.
+ * @param options The options to use for the upload.
+ */
+export declare function uploadTestTranslations(input: string, options: UploadTestTranslationOptions): Promise<UploadTestTranslationResult>;
+/**
+ * Generates the API files directly from the given translation and
+ * uploads them to the HelloAO test s3 bucket.
+ *
+ * Requires access to the HelloAO test s3 bucket. Email hello@helloao.org for access.
+ *
+ * @param input The input directory that the translations are stored in.
+ * @param options The options to use for the upload.
+ */
+export declare function uploadTestTranslation(input: string, options: UploadTestTranslationOptions): Promise<UploadTestTranslationResult>;
 //# sourceMappingURL=actions.d.ts.map

package/actions.js

@@ -31,6 +31,10 @@ exports.importTranslation = importTranslation;
 exports.importTranslations = importTranslations;
 exports.fetchTranslations = fetchTranslations;
 exports.fetchAudio = fetchAudio;
+exports.generateTranslationsFiles = generateTranslationsFiles;
+exports.generateTranslationFiles = generateTranslationFiles;
+exports.uploadTestTranslations = uploadTestTranslations;
+exports.uploadTestTranslation = uploadTestTranslation;
 const node_path_1 = __importStar(require("node:path"));
 const database = __importStar(require("./db"));
 const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
@@ -42,6 +46,11 @@ const fs_extra_1 = require("fs-extra");
 const audio_1 = require("@helloao/tools/generation/audio");
 const book_order_1 = require("@helloao/tools/generation/book-order");
 const downloads_1 = require("./downloads");
+const iterators_1 = require("@helloao/tools/parser/iterators");
+const files_1 = require("./files");
+const dataset_1 = require("@helloao/tools/generation/dataset");
+const uploads_1 = require("./uploads");
+const s3_1 = require("./s3");
 /**
  * Initializes a new Bible API DB.
  * @param dbPath The path to the database. If null or empty, then the "bible-api.db" will be used from the current working directory.
@@ -56,7 +65,9 @@ async function initDb(dbPath, options) {
             console.log('Copying schema from source DB...');
             if (options.language) {
                 console.log('Copying only the following languages:', options.language);
-                const languages = `(${options.language.map((l) => `'${l}'`).join(', ')})`;
+                const languages = `(${options.language
+                    .map((l) => `'${l}'`)
+                    .join(', ')})`;
                 db.exec(`
                     ATTACH DATABASE "${sourcePath}" AS source;
 
@@ -142,7 +153,7 @@ async function importTranslations(dir, options) {
     const db = await database.getDbFromDir(process.cwd());
     try {
         const files = await (0, promises_1.readdir)(dir);
-        const translationDirs = files.map(f => node_path_1.default.resolve(dir, f));
+        const translationDirs = files.map((f) => node_path_1.default.resolve(dir, f));
         console.log(`Importing ${translationDirs.length} translations`);
         await database.importTranslations(db, translationDirs, parser, !!options.overwrite);
     }
@@ -164,7 +175,9 @@ async function fetchTranslations(dir, translations, options = {}) {
     const collection = await client.fetch_collection();
     const collectionTranslations = collection.get_translations();
     console.log(`Discovered ${collectionTranslations.length} translations`);
-    const filtered = translations && translations.length <= 0 ? collectionTranslations : collectionTranslations.filter(t => translationsSet.has(t.id));
+    const filtered = translations && translations.length <= 0
+        ? collectionTranslations
+        : collectionTranslations.filter((t) => translationsSet.has(t.id));
     let batches = [];
     while (filtered.length > 0) {
         batches.push(filtered.splice(0, 10));
@@ -187,7 +200,8 @@ async function fetchTranslations(dir, translations, options = {}) {
             };
             const books = await Promise.all(collection.get_books(t.id).map(async (b) => {
                 const name = `${b.id}.usx`;
-                if (!options.all && await (0, fs_extra_1.exists)(node_path_1.default.resolve(dir, translation.id, name))) {
+                if (!options.all &&
+                    (await (0, fs_extra_1.exists)(node_path_1.default.resolve(dir, translation.id, name)))) {
                     return null;
                 }
                 const content = await collection.fetch_book(t.id, b.id, 'usx');
@@ -196,7 +210,7 @@ async function fetchTranslations(dir, translations, options = {}) {
                     fileType: 'usx',
                     content: contentString,
                     metadata: {
-                        translation
+                        translation,
                     },
                     name,
                 };
@@ -252,7 +266,7 @@ async function fetchAudio(dir, translations, options = {}) {
                 const [translationId, reader] = translation.split('/');
                 const name = `${chapter}.${reader}${ext}`;
                 const fullPath = node_path_1.default.resolve(dir, 'audio', translationId, bookId, name);
-                if (!options.all && await (0, fs_extra_1.exists)(fullPath)) {
+                if (!options.all && (await (0, fs_extra_1.exists)(fullPath))) {
                     continue;
                 }
                 await (0, downloads_1.downloadFile)(url, fullPath);
@@ -260,3 +274,105 @@ async function fetchAudio(dir, translations, options = {}) {
         }
     }
 }
+/**
+ * Generates the translation files directly from the translations stored in the given input directory.
+ * @param input The input directory that the translations are stored in.
+ * @param dest The destination to upload the API files to.
+ * @param options The options for the generation.
+ */
+async function generateTranslationsFiles(input, dest, options) {
+    const parser = new linkedom_1.DOMParser();
+    globalThis.DOMParser = linkedom_1.DOMParser;
+    globalThis.Element = linkedom_1.Element;
+    globalThis.Node = linkedom_1.Node;
+    const dirs = await (0, promises_1.readdir)(node_path_1.default.resolve(input));
+    const batchSize = typeof options.batchSize === 'number'
+        ? options.batchSize
+        : parseInt(options.batchSize);
+    for (let b of (0, iterators_1.batch)(dirs, batchSize)) {
+        const files = await (0, files_1.loadTranslationsFiles)(b);
+        const dataset = (0, dataset_1.generateDataset)(files, parser);
+        await (0, uploads_1.serializeAndUploadDatasets)(dest, (0, iterators_1.toAsyncIterable)([dataset]), options);
+    }
+}
+/**
+ * Generates the translation files directly from the translation stored in the given input directory.
+ * @param input The input directory that the translation is stored in.
+ * @param dest The destination to upload the API files to.
+ * @param options The options for the generation.
+ */
+async function generateTranslationFiles(input, dest, options) {
+    const parser = new linkedom_1.DOMParser();
+    globalThis.DOMParser = linkedom_1.DOMParser;
+    globalThis.Element = linkedom_1.Element;
+    globalThis.Node = linkedom_1.Node;
+    const files = await (0, files_1.loadTranslationFiles)(node_path_1.default.resolve(input));
+    const dataset = (0, dataset_1.generateDataset)(files, parser);
+    await (0, uploads_1.serializeAndUploadDatasets)(dest, (0, iterators_1.toAsyncIterable)([dataset]), options);
+}
+/**
+ * Generates the API files directly from the translations stored in the given input directory and
+ * uploads them to the HelloAO test s3 bucket.
+ *
+ * Requires access to the HelloAO test s3 bucket. Email hello@helloao.org for access.
+ *
+ * @param input The input directory that the translations are stored in.
+ * @param options The options to use for the upload.
+ */
+async function uploadTestTranslations(input, options) {
+    const parser = new linkedom_1.DOMParser();
+    globalThis.DOMParser = linkedom_1.DOMParser;
+    globalThis.Element = linkedom_1.Element;
+    globalThis.Node = linkedom_1.Node;
+    const dirs = await (0, promises_1.readdir)(node_path_1.default.resolve(input));
+    const files = await (0, files_1.loadTranslationsFiles)(dirs);
+    const hash = (0, files_1.hashInputFiles)(files);
+    const dataset = (0, dataset_1.generateDataset)(files, parser);
+    const url = options.s3Url || 's3://ao-bible-api-public-uploads';
+    await (0, uploads_1.serializeAndUploadDatasets)(url, (0, iterators_1.toAsyncIterable)([dataset]), {
+        ...options,
+        pathPrefix: `/${hash}`,
+    });
+    const urls = getUrls(url);
+    return {
+        ...urls,
+        version: hash,
+        availableTranslationsUrl: `${urls.url}/${hash}/api/available_translations.json`,
+    };
+}
+/**
+ * Generates the API files directly from the given translation and
+ * uploads them to the HelloAO test s3 bucket.
+ *
+ * Requires access to the HelloAO test s3 bucket. Email hello@helloao.org for access.
+ *
+ * @param input The input directory that the translations are stored in.
+ * @param options The options to use for the upload.
+ */
+async function uploadTestTranslation(input, options) {
+    const parser = new linkedom_1.DOMParser();
+    globalThis.DOMParser = linkedom_1.DOMParser;
+    globalThis.Element = linkedom_1.Element;
+    globalThis.Node = linkedom_1.Node;
+    const files = await (0, files_1.loadTranslationFiles)(node_path_1.default.resolve(input));
+    const hash = (0, files_1.hashInputFiles)(files);
+    const dataset = (0, dataset_1.generateDataset)(files, parser);
+    const url = options.s3Url || 's3://ao-bible-api-public-uploads';
+    await (0, uploads_1.serializeAndUploadDatasets)(url, (0, iterators_1.toAsyncIterable)([dataset]), {
+        ...options,
+        pathPrefix: `/${hash}`,
+    });
+    const urls = getUrls(url);
+    return {
+        ...urls,
+        version: hash,
+        availableTranslationsUrl: `${urls.url}/${hash}/api/available_translations.json`,
+    };
+}
+function getUrls(dest) {
+    const url = (0, s3_1.getHttpUrl)(dest);
+    return {
+        uploadS3Url: dest,
+        url: url,
+    };
+}

package/cli.js

@@ -11,38 +11,101 @@ const linkedom_1 = require("linkedom");
 const downloads_1 = require("./downloads");
 const uploads_1 = require("./uploads");
 const actions_1 = require("./actions");
-const files_1 = require("./files");
-const dataset_1 = require("@helloao/tools/generation/dataset");
-const iterators_1 = require("@helloao/tools/parser/iterators");
+const db_1 = require("./db");
+const prompts_1 = require("@inquirer/prompts");
 async function start() {
     const parser = new linkedom_1.DOMParser();
     globalThis.DOMParser = linkedom_1.DOMParser;
     globalThis.Element = linkedom_1.Element;
     globalThis.Node = linkedom_1.Node;
     const program = new commander_1.Command();
-    program.name('helloao')
+    program
+        .name('helloao')
         .description('A CLI for managing a Free Use Bible API.')
         .version('0.0.1');
-    program.command('init [path]')
+    program
+        .command('init [path]')
         .description('Initialize a new Bible API DB.')
         .option('--source <path>', 'The source database to copy from.')
         .option('--language <languages...>', 'The language(s) that the database should be initialized with.')
         .action(async (dbPath, options) => {
         await (0, actions_1.initDb)(dbPath, options);
     });
-    program.command('import-translation <dir> [dirs...]')
+    program
+        .command('import-translation <dir> [dirs...]')
         .description('Imports a translation from the given directory into the database.')
         .option('--overwrite', 'Whether to overwrite existing files.')
         .action(async (dir, dirs, options) => {
         await (0, actions_1.importTranslation)(dir, dirs, options);
     });
-    program.command('import-translations <dir>')
+    program
+        .command('import-translations <dir>')
         .description('Imports all translations from the given directory into the database.')
         .option('--overwrite', 'Whether to overwrite existing files.')
         .action(async (dir, options) => {
         await (0, actions_1.importTranslations)(dir, options);
     });
-    program.command('generate-translation-files <input> <dir>')
+    program
+        .command('upload-test-translation <input>')
+        .description(`Uploads a translation to the HelloAO Free Bible API test S3 bucket.\nRequires access to the HelloAO Free Bible API test S3 bucket.\nFor inquiries, please contact hello@helloao.org.`)
+        .option('--batch-size <size>', 'The number of translations to generate API files for in each batch.', '50')
+        .option('--translations <translations...>', 'The translations to generate API files for.')
+        .option('--overwrite', 'Whether to overwrite existing files.')
+        .option('--overwrite-common-files', 'Whether to overwrite only common files.')
+        .option('--file-pattern <pattern>', 'The file pattern regex that should be used to filter the files that are generated.')
+        .option('--use-common-name', 'Whether to use the common name for the book chapter API link. If false, then book IDs are used.')
+        .option('--generate-audio-files', 'Whether to replace the audio URLs in the dataset with ones that are hosted locally.')
+        .option('--profile <profile>', 'The AWS profile to use for uploading to S3.')
+        .option('--access-key-id <accessKeyId>', 'The AWS access key ID to use for uploading to S3.')
+        .option('--secret-access-key <secretAccessKey>', 'The AWS Secret Access Key to use for uploading to S3.')
+        .option('--pretty', 'Whether to generate pretty-printed JSON files.')
+        .option('--s3-url <s3Url>', 'The S3 bucket URL to upload the files to.', 's3://ao-bible-api-public-uploads')
+        .action(async (input, options) => {
+        const good = await (0, prompts_1.confirm)({
+            message: 'Uploaded files will be publicly accessible. Continue?',
+            default: false,
+        });
+        if (!good) {
+            return;
+        }
+        const result = await (0, actions_1.uploadTestTranslation)(input, options);
+        console.log('\n');
+        console.log('Version:               ', result.version);
+        console.log('Uploaded to:           ', result.uploadS3Url);
+        console.log('URL:                   ', result.url);
+        console.log('Available Translations:', result.availableTranslationsUrl);
+    });
+    program
+        .command('upload-test-translations <input>')
+        .description(`Uploads all the translations in the given input directory to the HelloAO Free Bible API test S3 bucket.\nRequires access to the HelloAO Free Bible API test S3 bucket.\nFor inquiries, please contact hello@helloao.org.`)
+        .option('--batch-size <size>', 'The number of translations to generate API files for in each batch.', '50')
+        .option('--translations <translations...>', 'The translations to generate API files for.')
+        .option('--overwrite', 'Whether to overwrite existing files.')
+        .option('--overwrite-common-files', 'Whether to overwrite only common files.')
+        .option('--file-pattern <pattern>', 'The file pattern regex that should be used to filter the files that are generated.')
+        .option('--use-common-name', 'Whether to use the common name for the book chapter API link. If false, then book IDs are used.')
+        .option('--generate-audio-files', 'Whether to replace the audio URLs in the dataset with ones that are hosted locally.')
+        .option('--profile <profile>', 'The AWS profile to use for uploading to S3.')
+        .option('--access-key-id <accessKeyId>', 'The AWS access key ID to use for uploading to S3.')
+        .option('--secret-access-key <secretAccessKey>', 'The AWS Secret Access Key to use for uploading to S3.')
+        .option('--pretty', 'Whether to generate pretty-printed JSON files.')
+        .option('--s3-url <s3Url>', 'The S3 bucket URL to upload the files to.', 's3://ao-bible-api-public-uploads')
+        .action(async (input, options) => {
+        const good = await (0, prompts_1.confirm)({
+            message: 'Uploaded files will be publicly accessible. Continue?',
+            default: false,
+        });
+        if (!good) {
+            return;
+        }
+        const result = await (0, actions_1.uploadTestTranslations)(input, options);
+        console.log('\nVersion:             ', result.version);
+        console.log('Uploaded to:          ', result.uploadS3Url);
+        console.log('URL:                  ', result.url);
+        console.log('Available Translations:', result.availableTranslationsUrl);
+    });
+    program
+        .command('generate-translation-files <input> <dir>')
         .description('Generates API files from the given input translation.')
         .option('--batch-size <size>', 'The number of translations to generate API files for in each batch.', '50')
         .option('--translations <translations...>', 'The translations to generate API files for.')
@@ -52,17 +115,14 @@ async function start() {
         .option('--use-common-name', 'Whether to use the common name for the book chapter API link. If false, then book IDs are used.')
         .option('--generate-audio-files', 'Whether to replace the audio URLs in the dataset with ones that are hosted locally.')
         .option('--profile <profile>', 'The AWS profile to use for uploading to S3.')
+        .option('--access-key-id <accessKeyId>', 'The AWS access key ID to use for uploading to S3.')
+        .option('--secret-access-key <secretAccessKey>', 'The AWS Secret Access Key to use for uploading to S3.')
         .option('--pretty', 'Whether to generate pretty-printed JSON files.')
         .action(async (input, dest, options) => {
-        const parser = new linkedom_1.DOMParser();
-        globalThis.DOMParser = linkedom_1.DOMParser;
-        globalThis.Element = linkedom_1.Element;
-        globalThis.Node = linkedom_1.Node;
-        const files = await (0, files_1.loadTranslationFiles)(path_1.default.resolve(input));
-        const dataset = (0, dataset_1.generateDataset)(files, parser);
-        await (0, uploads_1.uploadApiFiles)(path_1.default.resolve(dest), options, (0, iterators_1.toAsyncIterable)([dataset]));
+        await (0, actions_1.generateTranslationFiles)(input, dest, options);
     });
-    program.command('generate-translations-files <input> <dir>')
+    program
+        .command('generate-translations-files <input> <dir>')
         .description('Generates API files from the given input translations.')
         .option('--batch-size <size>', 'The number of translations to generate API files for in each batch.', '50')
         .option('--translations <translations...>', 'The translations to generate API files for.')
@@ -72,21 +132,14 @@ async function start() {
         .option('--use-common-name', 'Whether to use the common name for the book chapter API link. If false, then book IDs are used.')
         .option('--generate-audio-files', 'Whether to replace the audio URLs in the dataset with ones that are hosted locally.')
         .option('--profile <profile>', 'The AWS profile to use for uploading to S3.')
+        .option('--access-key-id <accessKeyId>', 'The AWS access key ID to use for uploading to S3.')
+        .option('--secret-access-key <secretAccessKey>', 'The AWS Secret Access Key to use for uploading to S3.')
         .option('--pretty', 'Whether to generate pretty-printed JSON files.')
         .action(async (input, dest, options) => {
-        const parser = new linkedom_1.DOMParser();
-        globalThis.DOMParser = linkedom_1.DOMParser;
-        globalThis.Element = linkedom_1.Element;
-        globalThis.Node = linkedom_1.Node;
-        const dirs = await (0, promises_1.readdir)(path_1.default.resolve(input));
-        const batchSize = parseInt(options.batchSize);
-        for (let b of (0, iterators_1.batch)(dirs, batchSize)) {
-            const files = await (0, files_1.loadTranslationsFiles)(b);
-            const dataset = (0, dataset_1.generateDataset)(files, parser);
-            await (0, uploads_1.uploadApiFiles)(dest, options, (0, iterators_1.toAsyncIterable)([dataset]));
-        }
+        await (0, actions_1.generateTranslationsFiles)(input, dest, options);
     });
-    program.command('upload-api-files')
+    program
+        .command('upload-api-files')
         .argument('<dest>', 'The destination to upload the API files to.')
         .description('Uploads API files to the specified destination. For S3, use the format s3://bucket-name/path/to/folder.')
         .option('--batch-size <size>', 'The number of translations to generate API files for in each batch.', '50')
@@ -97,23 +150,34 @@ async function start() {
         .option('--use-common-name', 'Whether to use the common name for the book chapter API link. If false, then book IDs are used.')
         .option('--generate-audio-files', 'Whether to replace the audio URLs in the dataset with ones that are hosted locally.')
         .option('--profile <profile>', 'The AWS profile to use for uploading to S3.')
+        .option('--access-key-id <accessKeyId>', 'The AWS access key ID to use for uploading to S3.')
+        .option('--secret-access-key <secretAccessKey>', 'The AWS Secret Access Key to use for uploading to S3.')
         .option('--pretty', 'Whether to generate pretty-printed JSON files.')
         .action(async (dest, options) => {
-        await (0, uploads_1.uploadApiFilesFromDatabase)(dest, options);
+        const db = (0, db_1.getPrismaDbFromDir)(process.cwd());
+        try {
+            await (0, uploads_1.uploadApiFilesFromDatabase)(db, dest, options);
+        }
+        finally {
+            db.$disconnect();
+        }
     });
-    program.command('fetch-translations <dir> [translations...]')
+    program
+        .command('fetch-translations <dir> [translations...]')
         .description('Fetches the specified translations from fetch.bible and places them in the given directory.')
         .option('-a, --all', 'Fetch all translations. If omitted, only undownloaded translations will be fetched.')
         .action(async (dir, translations, options) => {
         await (0, actions_1.fetchTranslations)(dir, translations, options);
     });
-    program.command('fetch-audio <dir> [translations...]')
+    program
+        .command('fetch-audio <dir> [translations...]')
         .description('Fetches the specified audio translations and places them in the given directory.\nTranslations should be in the format "translationId/audioId". e.g. "BSB/gilbert"')
         .option('-a, --all', 'Fetch all translations. If omitted, only undownloaded translations will be fetched.')
         .action(async (dir, translations, options) => {
         await (0, actions_1.fetchAudio)(dir, translations, options);
     });
-    program.command('fetch-bible-metadata <dir>')
+    program
+        .command('fetch-bible-metadata <dir>')
         .description('Fetches the Theographic bible metadata and places it in the given directory.')
         .action(async (dir) => {
         let files = [

package/db.d.ts

@@ -1,10 +1,10 @@
-import { PrismaClient } from "./prisma-gen";
+import { PrismaClient } from './prisma-gen';
 import { Database } from 'better-sqlite3';
-import { DatasetOutput, DatasetTranslation, DatasetTranslationBook } from "@helloao/tools/generation/dataset";
-import { InputFile, TranslationBookChapter } from "@helloao/tools/generation";
-import { GenerateApiOptions } from "@helloao/tools/generation/api";
-import { DOMParser } from "linkedom";
-import { Readable } from "stream";
+import { DatasetOutput, DatasetTranslation, DatasetTranslationBook } from '@helloao/tools/generation/dataset';
+import { InputFile, TranslationBookChapter } from '@helloao/tools/generation';
+import { GenerateApiOptions } from '@helloao/tools/generation/api';
+import { DOMParser } from 'linkedom';
+import { Readable } from 'stream';
 /**
  * Imports the translations from the given directories into the database.
  * @param db The database to import the translations into.
@@ -72,39 +72,21 @@ export interface SerializeApiOptions extends GenerateApiOptions {
     pretty?: boolean;
 }
 /**
- * Generates and serializes the API files for the dataset that is stored in the database.
+ * Generates and serializes the API files for the datasets that are stored in the database.
  * Yields each batch of serialized files.
  * @param db The database that the dataset should be loaded from.
- * @param options The options to use for generating the API.
+ * @param options The options to use for serializing the files.
+ * @param apiOptions The options to use for generating the API files.
  * @param translationsPerBatch The number of translations that should be loaded and written per batch.
  * @param translations The list of translations that should be loaded. If not provided, all translations will be loaded.
  */
-export declare function serializeFilesForDataset(db: PrismaClient, options: SerializeApiOptions, translationsPerBatch?: number, translations?: string[]): AsyncGenerator<SerializedFile[]>;
-/**
- * Serializes the API files for the given datasets.
- * @param datasets The dataasets to serialize.
- * @param options The options to use for serializing the files.
- */
-export declare function serializeFiles(datasets: AsyncIterable<DatasetOutput>, options: SerializeApiOptions): AsyncGenerator<SerializedFile[]>;
+export declare function serializeFilesFromDatabase(db: PrismaClient, options?: SerializeApiOptions, translationsPerBatch?: number, translations?: string[]): AsyncGenerator<SerializedFile[]>;
 /**
- * Defines an interface that contains information about a serialized file.
+ * Generates and serializes the API files for the given datasets.
+ * Yields each batch of serialized files.
+ *
+ * @param datasets The datasets to serialize.
+ * @param options The options to use for generating and serializing the files.
  */
-export interface Uploader {
-    /**
-     * Gets the ideal batch size for the uploader.
-     * Null if the uploader does not need batching.
-     */
-    idealBatchSize: number | null;
-    /**
-     * Uploads the given file.
-     * @param file The file to upload.
-     * @param overwrite Whether the file should be overwritten if it already exists.
-     * @returns True if the file was uploaded. False if the file was skipped due to already existing.
-     */
-    upload(file: SerializedFile, overwrite: boolean): Promise<boolean>;
-    /**
-     * Disposes resources that the uploader uses.
-     */
-    dispose?(): Promise<void>;
-}
+export declare function serializeDatasets(datasets: AsyncIterable<DatasetOutput>, options?: SerializeApiOptions): AsyncGenerator<SerializedFile[]>;
 //# sourceMappingURL=db.d.ts.map