@helloao/cli 0.0.5 → 0.0.6

package/cli.js

@@ -11,39 +11,101 @@ const linkedom_1 = require("linkedom");
 const downloads_1 = require("./downloads");
 const uploads_1 = require("./uploads");
 const actions_1 = require("./actions");
-const files_1 = require("./files");
-const dataset_1 = require("@helloao/tools/generation/dataset");
-const iterators_1 = require("@helloao/tools/parser/iterators");
 const db_1 = require("./db");
+const prompts_1 = require("@inquirer/prompts");
 async function start() {
     const parser = new linkedom_1.DOMParser();
     globalThis.DOMParser = linkedom_1.DOMParser;
     globalThis.Element = linkedom_1.Element;
     globalThis.Node = linkedom_1.Node;
     const program = new commander_1.Command();
-    program.name('helloao')
+    program
+        .name('helloao')
         .description('A CLI for managing a Free Use Bible API.')
         .version('0.0.1');
-    program.command('init [path]')
+    program
+        .command('init [path]')
         .description('Initialize a new Bible API DB.')
         .option('--source <path>', 'The source database to copy from.')
         .option('--language <languages...>', 'The language(s) that the database should be initialized with.')
         .action(async (dbPath, options) => {
         await (0, actions_1.initDb)(dbPath, options);
     });
-    program.command('import-translation <dir> [dirs...]')
+    program
+        .command('import-translation <dir> [dirs...]')
         .description('Imports a translation from the given directory into the database.')
         .option('--overwrite', 'Whether to overwrite existing files.')
         .action(async (dir, dirs, options) => {
         await (0, actions_1.importTranslation)(dir, dirs, options);
     });
-    program.command('import-translations <dir>')
+    program
+        .command('import-translations <dir>')
         .description('Imports all translations from the given directory into the database.')
         .option('--overwrite', 'Whether to overwrite existing files.')
         .action(async (dir, options) => {
         await (0, actions_1.importTranslations)(dir, options);
     });
-    program.command('generate-translation-files <input> <dir>')
+    program
+        .command('upload-test-translation <input>')
+        .description(`Uploads a translation to the HelloAO Free Bible API test S3 bucket.\nRequires access to the HelloAO Free Bible API test S3 bucket.\nFor inquiries, please contact hello@helloao.org.`)
+        .option('--batch-size <size>', 'The number of translations to generate API files for in each batch.', '50')
+        .option('--translations <translations...>', 'The translations to generate API files for.')
+        .option('--overwrite', 'Whether to overwrite existing files.')
+        .option('--overwrite-common-files', 'Whether to overwrite only common files.')
+        .option('--file-pattern <pattern>', 'The file pattern regex that should be used to filter the files that are generated.')
+        .option('--use-common-name', 'Whether to use the common name for the book chapter API link. If false, then book IDs are used.')
+        .option('--generate-audio-files', 'Whether to replace the audio URLs in the dataset with ones that are hosted locally.')
+        .option('--profile <profile>', 'The AWS profile to use for uploading to S3.')
+        .option('--access-key-id <accessKeyId>', 'The AWS access key ID to use for uploading to S3.')
+        .option('--secret-access-key <secretAccessKey>', 'The AWS Secret Access Key to use for uploading to S3.')
+        .option('--pretty', 'Whether to generate pretty-printed JSON files.')
+        .option('--s3-url <s3Url>', 'The S3 bucket URL to upload the files to.', 's3://ao-bible-api-public-uploads')
+        .action(async (input, options) => {
+        const good = await (0, prompts_1.confirm)({
+            message: 'Uploaded files will be publicly accessible. Continue?',
+            default: false,
+        });
+        if (!good) {
+            return;
+        }
+        const result = await (0, actions_1.uploadTestTranslation)(input, options);
+        console.log('\n');
+        console.log('Version:               ', result.version);
+        console.log('Uploaded to:           ', result.uploadS3Url);
+        console.log('URL:                   ', result.url);
+        console.log('Available Translations:', result.availableTranslationsUrl);
+    });
+    program
+        .command('upload-test-translations <input>')
+        .description(`Uploads all the translations in the given input directory to the HelloAO Free Bible API test S3 bucket.\nRequires access to the HelloAO Free Bible API test S3 bucket.\nFor inquiries, please contact hello@helloao.org.`)
+        .option('--batch-size <size>', 'The number of translations to generate API files for in each batch.', '50')
+        .option('--translations <translations...>', 'The translations to generate API files for.')
+        .option('--overwrite', 'Whether to overwrite existing files.')
+        .option('--overwrite-common-files', 'Whether to overwrite only common files.')
+        .option('--file-pattern <pattern>', 'The file pattern regex that should be used to filter the files that are generated.')
+        .option('--use-common-name', 'Whether to use the common name for the book chapter API link. If false, then book IDs are used.')
+        .option('--generate-audio-files', 'Whether to replace the audio URLs in the dataset with ones that are hosted locally.')
+        .option('--profile <profile>', 'The AWS profile to use for uploading to S3.')
+        .option('--access-key-id <accessKeyId>', 'The AWS access key ID to use for uploading to S3.')
+        .option('--secret-access-key <secretAccessKey>', 'The AWS Secret Access Key to use for uploading to S3.')
+        .option('--pretty', 'Whether to generate pretty-printed JSON files.')
+        .option('--s3-url <s3Url>', 'The S3 bucket URL to upload the files to.', 's3://ao-bible-api-public-uploads')
+        .action(async (input, options) => {
+        const good = await (0, prompts_1.confirm)({
+            message: 'Uploaded files will be publicly accessible. Continue?',
+            default: false,
+        });
+        if (!good) {
+            return;
+        }
+        const result = await (0, actions_1.uploadTestTranslations)(input, options);
+        console.log('\nVersion:             ', result.version);
+        console.log('Uploaded to:          ', result.uploadS3Url);
+        console.log('URL:                  ', result.url);
+        console.log('Available Translations:', result.availableTranslationsUrl);
+    });
+    program
+        .command('generate-translation-files <input> <dir>')
         .description('Generates API files from the given input translation.')
         .option('--batch-size <size>', 'The number of translations to generate API files for in each batch.', '50')
         .option('--translations <translations...>', 'The translations to generate API files for.')
@@ -53,17 +115,14 @@ async function start() {
         .option('--use-common-name', 'Whether to use the common name for the book chapter API link. If false, then book IDs are used.')
         .option('--generate-audio-files', 'Whether to replace the audio URLs in the dataset with ones that are hosted locally.')
         .option('--profile <profile>', 'The AWS profile to use for uploading to S3.')
+        .option('--access-key-id <accessKeyId>', 'The AWS access key ID to use for uploading to S3.')
+        .option('--secret-access-key <secretAccessKey>', 'The AWS Secret Access Key to use for uploading to S3.')
         .option('--pretty', 'Whether to generate pretty-printed JSON files.')
         .action(async (input, dest, options) => {
-        const parser = new linkedom_1.DOMParser();
-        globalThis.DOMParser = linkedom_1.DOMParser;
-        globalThis.Element = linkedom_1.Element;
-        globalThis.Node = linkedom_1.Node;
-        const files = await (0, files_1.loadTranslationFiles)(path_1.default.resolve(input));
-        const dataset = (0, dataset_1.generateDataset)(files, parser);
-        await (0, uploads_1.serializeAndUploadDatasets)(path_1.default.resolve(dest), (0, iterators_1.toAsyncIterable)([dataset]), options);
+        await (0, actions_1.generateTranslationFiles)(input, dest, options);
     });
-    program.command('generate-translations-files <input> <dir>')
+    program
+        .command('generate-translations-files <input> <dir>')
         .description('Generates API files from the given input translations.')
         .option('--batch-size <size>', 'The number of translations to generate API files for in each batch.', '50')
         .option('--translations <translations...>', 'The translations to generate API files for.')
@@ -73,21 +132,14 @@ async function start() {
         .option('--use-common-name', 'Whether to use the common name for the book chapter API link. If false, then book IDs are used.')
         .option('--generate-audio-files', 'Whether to replace the audio URLs in the dataset with ones that are hosted locally.')
         .option('--profile <profile>', 'The AWS profile to use for uploading to S3.')
+        .option('--access-key-id <accessKeyId>', 'The AWS access key ID to use for uploading to S3.')
+        .option('--secret-access-key <secretAccessKey>', 'The AWS Secret Access Key to use for uploading to S3.')
         .option('--pretty', 'Whether to generate pretty-printed JSON files.')
         .action(async (input, dest, options) => {
-        const parser = new linkedom_1.DOMParser();
-        globalThis.DOMParser = linkedom_1.DOMParser;
-        globalThis.Element = linkedom_1.Element;
-        globalThis.Node = linkedom_1.Node;
-        const dirs = await (0, promises_1.readdir)(path_1.default.resolve(input));
-        const batchSize = parseInt(options.batchSize);
-        for (let b of (0, iterators_1.batch)(dirs, batchSize)) {
-            const files = await (0, files_1.loadTranslationsFiles)(b);
-            const dataset = (0, dataset_1.generateDataset)(files, parser);
-            await (0, uploads_1.serializeAndUploadDatasets)(dest, (0, iterators_1.toAsyncIterable)([dataset]), options);
-        }
+        await (0, actions_1.generateTranslationsFiles)(input, dest, options);
     });
-    program.command('upload-api-files')
+    program
+        .command('upload-api-files')
         .argument('<dest>', 'The destination to upload the API files to.')
         .description('Uploads API files to the specified destination. For S3, use the format s3://bucket-name/path/to/folder.')
         .option('--batch-size <size>', 'The number of translations to generate API files for in each batch.', '50')
@@ -98,6 +150,8 @@ async function start() {
         .option('--use-common-name', 'Whether to use the common name for the book chapter API link. If false, then book IDs are used.')
         .option('--generate-audio-files', 'Whether to replace the audio URLs in the dataset with ones that are hosted locally.')
         .option('--profile <profile>', 'The AWS profile to use for uploading to S3.')
+        .option('--access-key-id <accessKeyId>', 'The AWS access key ID to use for uploading to S3.')
+        .option('--secret-access-key <secretAccessKey>', 'The AWS Secret Access Key to use for uploading to S3.')
         .option('--pretty', 'Whether to generate pretty-printed JSON files.')
         .action(async (dest, options) => {
         const db = (0, db_1.getPrismaDbFromDir)(process.cwd());
@@ -108,19 +162,22 @@ async function start() {
             db.$disconnect();
         }
     });
-    program.command('fetch-translations <dir> [translations...]')
+    program
+        .command('fetch-translations <dir> [translations...]')
         .description('Fetches the specified translations from fetch.bible and places them in the given directory.')
         .option('-a, --all', 'Fetch all translations. If omitted, only undownloaded translations will be fetched.')
         .action(async (dir, translations, options) => {
         await (0, actions_1.fetchTranslations)(dir, translations, options);
     });
-    program.command('fetch-audio <dir> [translations...]')
+    program
+        .command('fetch-audio <dir> [translations...]')
         .description('Fetches the specified audio translations and places them in the given directory.\nTranslations should be in the format "translationId/audioId". e.g. "BSB/gilbert"')
         .option('-a, --all', 'Fetch all translations. If omitted, only undownloaded translations will be fetched.')
         .action(async (dir, translations, options) => {
         await (0, actions_1.fetchAudio)(dir, translations, options);
     });
-    program.command('fetch-bible-metadata <dir>')
+    program
+        .command('fetch-bible-metadata <dir>')
         .description('Fetches the Theographic bible metadata and places it in the given directory.')
         .action(async (dir) => {
         let files = [

package/db.d.ts

@@ -1,10 +1,10 @@
-import { PrismaClient } from "./prisma-gen";
+import { PrismaClient } from './prisma-gen';
 import { Database } from 'better-sqlite3';
-import { DatasetOutput, DatasetTranslation, DatasetTranslationBook } from "@helloao/tools/generation/dataset";
-import { InputFile, TranslationBookChapter } from "@helloao/tools/generation";
-import { SerializeApiOptions, SerializedFile } from "./files";
-import { DOMParser } from "linkedom";
-import { GenerateApiOptions } from "@helloao/tools/generation/api";
+import { DatasetOutput, DatasetTranslation, DatasetTranslationBook } from '@helloao/tools/generation/dataset';
+import { InputFile, TranslationBookChapter } from '@helloao/tools/generation';
+import { GenerateApiOptions } from '@helloao/tools/generation/api';
+import { DOMParser } from 'linkedom';
+import { Readable } from 'stream';
 /**
  * Imports the translations from the given directories into the database.
  * @param db The database to import the translations into.
@@ -50,6 +50,14 @@ export declare function getPrismaDbFromDir(dir: string): PrismaClient<{
 }, never, import("prisma-gen/runtime/library").DefaultArgs>;
 export declare function getDbFromDir(dir: string): Promise<Database>;
 export declare function getDb(dbPath: string): Promise<Database>;
+export interface SerializedFile {
+    path: string;
+    content: string | Readable;
+    /**
+     * Gets the base64-encoded SHA256 hash of the content of the file.
+     */
+    sha256?(): string;
+}
 /**
  * Loads the datasets from the database in a series of batches.
  * @param db The database.
@@ -57,7 +65,12 @@ export declare function getDb(dbPath: string): Promise<Database>;
  * @param translationsToLoad The list of translations to load. If not provided, all translations will be loaded.
  */
 export declare function loadDatasets(db: PrismaClient, translationsPerBatch?: number, translationsToLoad?: string[]): AsyncGenerator<DatasetOutput>;
-export type SerializeDatasetOptions = SerializeApiOptions & GenerateApiOptions;
+export interface SerializeApiOptions extends GenerateApiOptions {
+    /**
+     * Whether the output should be pretty-printed.
+     */
+    pretty?: boolean;
+}
 /**
  * Generates and serializes the API files for the datasets that are stored in the database.
  * Yields each batch of serialized files.
@@ -67,7 +80,7 @@ export type SerializeDatasetOptions = SerializeApiOptions & GenerateApiOptions;
  * @param translationsPerBatch The number of translations that should be loaded and written per batch.
  * @param translations The list of translations that should be loaded. If not provided, all translations will be loaded.
  */
-export declare function serializeFilesFromDatabase(db: PrismaClient, options?: SerializeDatasetOptions, translationsPerBatch?: number, translations?: string[]): AsyncGenerator<SerializedFile[]>;
+export declare function serializeFilesFromDatabase(db: PrismaClient, options?: SerializeApiOptions, translationsPerBatch?: number, translations?: string[]): AsyncGenerator<SerializedFile[]>;
 /**
  * Generates and serializes the API files for the given datasets.
  * Yields each batch of serialized files.
@@ -75,5 +88,5 @@ export declare function serializeFilesFromDatabase(db: PrismaClient, options?: S
  * @param datasets The datasets to serialize.
  * @param options The options to use for generating and serializing the files.
  */
-export declare function serializeDatasets(datasets: AsyncIterable<DatasetOutput>, options?: SerializeDatasetOptions): AsyncGenerator<SerializedFile[]>;
+export declare function serializeDatasets(datasets: AsyncIterable<DatasetOutput>, options?: SerializeApiOptions): AsyncGenerator<SerializedFile[]>;
 //# sourceMappingURL=db.d.ts.map