@helloao/cli 0.0.4 → 0.0.6

package/db.js

@@ -40,8 +40,8 @@ exports.getPrismaDbFromDir = getPrismaDbFromDir;
 exports.getDbFromDir = getDbFromDir;
 exports.getDb = getDb;
 exports.loadDatasets = loadDatasets;
-exports.serializeFilesForDataset = serializeFilesForDataset;
-exports.serializeFiles = serializeFiles;
+exports.serializeFilesFromDatabase = serializeFilesFromDatabase;
+exports.serializeDatasets = serializeDatasets;
 const prisma_gen_1 = require("./prisma-gen");
 const path_1 = __importStar(require("path"));
 const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
@@ -51,10 +51,6 @@ const dataset_1 = require("@helloao/tools/generation/dataset");
 const api_1 = require("@helloao/tools/generation/api");
 const files_1 = require("./files");
 const hash_js_1 = require("hash.js");
-const lodash_1 = require("lodash");
-const path_2 = require("path");
-const stream_1 = require("stream");
-const base64_js_1 = require("base64-js");
 const cliPath = require.resolve('./index');
 const migrationsPath = path_1.default.resolve((0, path_1.dirname)(cliPath), 'migrations');
 /**
@@ -105,7 +101,9 @@ async function importTranslationFileBatch(db, files, parser, overwrite) {
     if (overwrite) {
         console.log('Overwriting existing translations.');
     }
-    const changedFiles = overwrite ? files : getChangedOrNewInputFiles(db, files);
+    const changedFiles = overwrite
+        ? files
+        : getChangedOrNewInputFiles(db, files);
     console.log('Processing', changedFiles.length, 'changed files');
     console.log('Skipping', files.length - changedFiles.length, 'unchanged files');
     const output = (0, dataset_1.generateDataset)(changedFiles, parser);
@@ -122,7 +120,7 @@ async function importTranslationFileBatch(db, files, parser, overwrite) {
  */
 function getChangedOrNewInputFiles(db, files) {
     const fileExists = db.prepare('SELECT COUNT(*) as c FROM InputFile WHERE translationId = @translationId AND name = @name AND sha256 = @sha256;');
-    return files.filter(f => {
+    return files.filter((f) => {
         const count = fileExists.get({
             translationId: f.metadata.translation.id,
             name: path_1.default.basename(f.name),
@@ -244,7 +242,7 @@ function insertTranslationBooks(db, translation, translationBooks) {
                 name: book.name,
                 commonName: book.commonName,
                 numberOfChapters: book.chapters.length,
-                bookOrder: book.order ?? 9999
+                bookOrder: book.order ?? 9999,
             });
         }
     });
@@ -431,7 +429,7 @@ function updateTranslationHashes(db, translations) {
         for (let book of books) {
             const chapters = getChapters.all({
                 translationId: translation.id,
-                bookId: book.id
+                bookId: book.id,
             });
             const bookSha = (0, hash_js_1.sha256)()
                 .update(book.translationId)
@@ -457,7 +455,7 @@ function updateTranslationHashes(db, translations) {
                         sha256: chapter.sha256,
                         translationId: chapter.translationId,
                         bookId: chapter.bookId,
-                        chapterNumber: chapter.number
+                        chapterNumber: chapter.number,
                     });
                 }
             });
@@ -471,7 +469,7 @@ function updateTranslationHashes(db, translations) {
                 updateBookHash.run({
                     sha256: book.sha256,
                     translationId: book.translationId,
-                    bookId: book.id
+                    bookId: book.id,
                 });
             }
         });
@@ -483,7 +481,7 @@ function updateTranslationHashes(db, translations) {
         for (let translation of translations) {
             updateTranslationHash.run({
                 sha256: translation.sha256,
-                translationId: translation.id
+                translationId: translation.id,
             });
         }
     });
@@ -506,8 +504,8 @@ function getPrismaDbFromDir(dir) {
         datasources: {
             db: {
                 url: `file:${dbPath}`,
-            }
-        }
+            },
+        },
     });
     return prisma;
 }
@@ -529,10 +527,12 @@ async function getDb(dbPath) {
         "applied_steps_count"   INTEGER UNSIGNED NOT NULL DEFAULT 0
     );`);
     const migrations = await (0, fs_extra_1.readdir)(migrationsPath);
-    const appliedMigrations = db.prepare('SELECT * FROM _prisma_migrations;').all();
+    const appliedMigrations = db
+        .prepare('SELECT * FROM _prisma_migrations;')
+        .all();
     let missingMigrations = [];
     for (let migration of migrations) {
-        if (appliedMigrations.some(m => m.migration_name === migration)) {
+        if (appliedMigrations.some((m) => m.migration_name === migration)) {
             continue;
         }
         if (path_1.default.extname(migration) !== '') {
@@ -574,7 +574,7 @@ async function* loadDatasets(db, translationsPerBatch = 50, translationsToLoad)
             query.where = {
                 id: {
                     in: translationsToLoad,
-                }
+                },
             };
         }
         const translations = await db.translation.findMany(query);
@@ -582,7 +582,7 @@ async function* loadDatasets(db, translationsPerBatch = 50, translationsToLoad)
             break;
         }
         const dataset = {
-            translations: []
+            translations: [],
         };
         for (let translation of translations) {
             const datasetTranslation = {
@@ -613,22 +613,19 @@ async function* loadDatasets(db, translationsPerBatch = 50, translationsToLoad)
                 const audioLinks = await db.chapterAudioUrl.findMany({
                     where: {
                         translationId: translation.id,
-                        bookId: book.id
+                        bookId: book.id,
                     },
-                    orderBy: [
-                        { number: 'asc' },
-                        { reader: 'asc' }
-                    ]
+                    orderBy: [{ number: 'asc' }, { reader: 'asc' }],
                 });
-                const bookChapters = chapters.map(chapter => {
+                const bookChapters = chapters.map((chapter) => {
                     return {
                         chapter: JSON.parse(chapter.json),
                         thisChapterAudioLinks: audioLinks
-                            .filter(link => link.number === chapter.number)
+                            .filter((link) => link.number === chapter.number)
                             .reduce((acc, link) => {
                             acc[link.reader] = link.url;
                             return acc;
-                        }, {})
+                        }, {}),
                     };
                 });
                 const datasetBook = {
@@ -643,112 +640,24 @@ async function* loadDatasets(db, translationsPerBatch = 50, translationsToLoad)
     }
 }
 /**
- * Generates and serializes the API files for the dataset that is stored in the database.
+ * Generates and serializes the API files for the datasets that are stored in the database.
  * Yields each batch of serialized files.
  * @param db The database that the dataset should be loaded from.
- * @param options The options to use for generating the API.
+ * @param options The options to use for serializing the files.
+ * @param apiOptions The options to use for generating the API files.
  * @param translationsPerBatch The number of translations that should be loaded and written per batch.
  * @param translations The list of translations that should be loaded. If not provided, all translations will be loaded.
  */
-async function* serializeFilesForDataset(db, options, translationsPerBatch = 50, translations) {
-    yield* serializeFiles(loadDatasets(db, translationsPerBatch, translations), options);
+function serializeFilesFromDatabase(db, options = {}, translationsPerBatch = 50, translations) {
+    return serializeDatasets(loadDatasets(db, translationsPerBatch, translations), options);
 }
 /**
- * Serializes the API files for the given datasets.
- * @param datasets The dataasets to serialize.
- * @param options The options to use for serializing the files.
+ * Generates and serializes the API files for the given datasets.
+ * Yields each batch of serialized files.
+ *
+ * @param datasets The datasets to serialize.
+ * @param options The options to use for generating and serializing the files.
  */
-async function* serializeFiles(datasets, options) {
-    const mergableFiles = new Map();
-    for await (let dataset of datasets) {
-        const api = (0, api_1.generateApiForDataset)(dataset, options);
-        const files = (0, api_1.generateFilesForApi)(api);
-        console.log('Generated', files.length, 'files');
-        let serializedFiles = [];
-        for (let file of files) {
-            if (file.mergable) {
-                let arr = mergableFiles.get(file.path);
-                if (!arr) {
-                    arr = [];
-                    mergableFiles.set(file.path, arr);
-                }
-                arr.push(file);
-                continue;
-            }
-            const serialized = await transformFile(file.path, file.content);
-            if (serialized) {
-                serializedFiles.push(serialized);
-            }
-        }
-        yield serializedFiles;
-    }
-    let serializedFiles = [];
-    for (let [path, files] of mergableFiles) {
-        let content = {};
-        for (let file of files) {
-            if (!content) {
-                content = file.content;
-            }
-            else {
-                content = (0, lodash_1.mergeWith)(content, file.content, (objValue, srcValue) => {
-                    if (Array.isArray(objValue)) {
-                        return objValue.concat(srcValue);
-                    }
-                    return undefined;
-                });
-            }
-        }
-        if (content) {
-            const serialized = await transformFile(path, content);
-            if (serialized) {
-                serializedFiles.push(serialized);
-            }
-        }
-    }
-    yield serializedFiles;
-    async function transformFile(path, content) {
-        let fileContent;
-        if (typeof content === 'function') {
-            fileContent = await content();
-        }
-        else {
-            fileContent = content;
-        }
-        const ext = (0, path_2.extname)(path);
-        if (ext === '.json') {
-            let json;
-            if (fileContent instanceof ReadableStream) {
-                json = '';
-                for await (const chunk of stream_1.Readable.fromWeb(fileContent, {
-                    encoding: 'utf-8'
-                })) {
-                    json += chunk;
-                }
-            }
-            else {
-                json = JSON.stringify(content, undefined, options.pretty ? 2 : undefined);
-            }
-            return {
-                path,
-                content: json,
-                sha256: () => (0, base64_js_1.fromByteArray)(new Uint8Array((0, hash_js_1.sha256)().update(json).digest()))
-            };
-        }
-        else if (ext === '.mp3') {
-            if (fileContent instanceof ReadableStream) {
-                return {
-                    path,
-                    content: stream_1.Readable.fromWeb(fileContent),
-                };
-            }
-            else {
-                console.warn('Expected content to be a readable stream for', path);
-                console.warn('Skipping file');
-                return null;
-            }
-        }
-        console.warn('Unknown file type', path);
-        console.warn('Skipping file');
-        return null;
-    }
+function serializeDatasets(datasets, options = {}) {
+    return (0, files_1.serializeOutputFiles)((0, api_1.generateOutputFilesFromDatasets)(datasets, options), options);
 }

package/files.d.ts

@@ -1,5 +1,62 @@
-import { InputFile } from "@helloao/tools/generation/common-types";
-import { SerializedFile, Uploader } from "./db";
+import { InputFile, OutputFile } from '@helloao/tools/generation/common-types';
+import { Readable } from 'stream';
+/**
+ * Defines an interface that contains information about a serialized file.
+ */
+export interface Uploader {
+    /**
+     * Gets the ideal batch size for the uploader.
+     * Null if the uploader does not need batching.
+     */
+    idealBatchSize: number | null;
+    /**
+     * Uploads the given file.
+     * @param file The file to upload.
+     * @param overwrite Whether the file should be overwritten if it already exists.
+     * @returns True if the file was uploaded. False if the file was skipped due to already existing.
+     */
+    upload(file: SerializedFile, overwrite: boolean): Promise<boolean>;
+    /**
+     * Disposes resources that the uploader uses.
+     */
+    dispose?(): Promise<void>;
+}
+/**
+ * Defines an interface for a file that has been serialized.
+ */
+export interface SerializedFile {
+    path: string;
+    content: string | Readable;
+    /**
+     * Gets the base64-encoded SHA256 hash of the content of the file.
+     */
+    sha256?(): string;
+}
+/**
+ * The options for serializing API files.
+ */
+export interface SerializeApiOptions {
+    /**
+     * Whether the output should be pretty-printed.
+     */
+    pretty?: boolean;
+}
+/**
+ * Serializes the given output files into serialized files using the given options.
+ *
+ * Each iteration of the given files will be processed as a batch, and any mergable files will automatically be merged together and serialized in the final batch.
+ *
+ * @param files The files that should be serialized.
+ * @param options The options for serialization.
+ */
+export declare function serializeOutputFiles(files: AsyncIterable<OutputFile[]>, options: SerializeApiOptions): AsyncGenerator<SerializedFile[]>;
+/**
+ * Serializes the given output file content into a serialized file.
+ * @param path The path that the file should be saved to.
+ * @param content The content of the file.
+ * @param options The options for serialization.
+ */
+export declare function serializeFile(path: string, content: OutputFile['content'], options: SerializeApiOptions): Promise<SerializedFile | null>;
 /**
  * Loads the files for the given translations.
  * @param dir The directory that the translations exist in.
@@ -53,4 +110,9 @@ export declare class ZipUploader implements Uploader {
     upload(file: SerializedFile, _overwrite: boolean): Promise<boolean>;
     dispose(): Promise<void>;
 }
+/**
+ * Calculates the SHa256 hash of the given input files.
+ * @param files The files to hash.
+ */
+export declare function hashInputFiles(files: InputFile[]): string;
 //# sourceMappingURL=files.d.ts.map

package/files.js

@@ -24,8 +24,11 @@ var __importStar = (this && this.__importStar) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ZipUploader = exports.FilesUploader = void 0;
+exports.serializeOutputFiles = serializeOutputFiles;
+exports.serializeFile = serializeFile;
 exports.loadTranslationsFiles = loadTranslationsFiles;
 exports.loadTranslationFiles = loadTranslationFiles;
+exports.hashInputFiles = hashInputFiles;
 const promises_1 = require("fs/promises");
 const path_1 = require("path");
 const path = __importStar(require("path"));
@@ -34,6 +37,113 @@ const zip_js_1 = require("@zip.js/zip.js");
 const stream_1 = require("stream");
 const hash_js_1 = require("hash.js");
 const usx_parser_1 = require("@helloao/tools/parser/usx-parser");
+const lodash_1 = require("lodash");
+const base64_js_1 = require("base64-js");
+/**
+ * Serializes the given output files into serialized files using the given options.
+ *
+ * Each iteration of the given files will be processed as a batch, and any mergable files will automatically be merged together and serialized in the final batch.
+ *
+ * @param files The files that should be serialized.
+ * @param options The options for serialization.
+ */
+async function* serializeOutputFiles(files, options) {
+    const mergableFiles = new Map();
+    for await (let batch of files) {
+        let serializedFiles = [];
+        for (let file of batch) {
+            if (file.mergable) {
+                let arr = mergableFiles.get(file.path);
+                if (!arr) {
+                    arr = [];
+                    mergableFiles.set(file.path, arr);
+                }
+                arr.push(file);
+                continue;
+            }
+            const serialized = await serializeFile(file.path, file.content, options);
+            if (serialized) {
+                serializedFiles.push(serialized);
+            }
+        }
+        yield serializedFiles;
+    }
+    let serializedFiles = [];
+    for (let [path, files] of mergableFiles) {
+        let content = {};
+        for (let file of files) {
+            if (!content) {
+                content = file.content;
+            }
+            else {
+                content = (0, lodash_1.mergeWith)(content, file.content, (objValue, srcValue) => {
+                    if (Array.isArray(objValue)) {
+                        return objValue.concat(srcValue);
+                    }
+                    return undefined;
+                });
+            }
+        }
+        if (content) {
+            const serialized = await serializeFile(path, content, options);
+            if (serialized) {
+                serializedFiles.push(serialized);
+            }
+        }
+    }
+    yield serializedFiles;
+}
+/**
+ * Serializes the given output file content into a serialized file.
+ * @param path The path that the file should be saved to.
+ * @param content The content of the file.
+ * @param options The options for serialization.
+ */
+async function serializeFile(path, content, options) {
+    let fileContent;
+    if (typeof content === 'function') {
+        fileContent = await content();
+    }
+    else {
+        fileContent = content;
+    }
+    const ext = (0, path_1.extname)(path);
+    if (ext === '.json') {
+        let json;
+        if (fileContent instanceof ReadableStream) {
+            json = '';
+            for await (const chunk of stream_1.Readable.fromWeb(fileContent, {
+                encoding: 'utf-8',
+            })) {
+                json += chunk;
+            }
+        }
+        else {
+            json = JSON.stringify(content, undefined, options.pretty ? 2 : undefined);
+        }
+        return {
+            path,
+            content: json,
+            sha256: () => (0, base64_js_1.fromByteArray)(new Uint8Array((0, hash_js_1.sha256)().update(json).digest())),
+        };
+    }
+    else if (ext === '.mp3') {
+        if (fileContent instanceof ReadableStream) {
+            return {
+                path,
+                content: stream_1.Readable.fromWeb(fileContent),
+            };
+        }
+        else {
+            console.warn('Expected content to be a readable stream for', path);
+            console.warn('Skipping file');
+            return null;
+        }
+    }
+    console.warn('Unknown file type', path);
+    console.warn('Skipping file');
+    return null;
+}
 /**
  * Loads the files for the given translations.
  * @param dir The directory that the translations exist in.
@@ -60,12 +170,14 @@ async function loadTranslationFiles(translation) {
         return [];
     }
     let files = await (0, promises_1.readdir)(translation);
-    let usfmFiles = files.filter(f => (0, path_1.extname)(f) === '.usfm' || (0, path_1.extname)(f) === '.usx' || (0, path_1.extname)(f) === '.json');
+    let usfmFiles = files.filter((f) => (0, path_1.extname)(f) === '.usfm' ||
+        (0, path_1.extname)(f) === '.usx' ||
+        (0, path_1.extname)(f) === '.json');
     if (usfmFiles.length <= 0) {
         translation = path.resolve(translation, 'usfm');
         if ((0, fs_extra_1.existsSync)(translation)) {
             files = await (0, promises_1.readdir)(translation);
-            usfmFiles = files.filter(f => (0, path_1.extname)(f) === '.usfm');
+            usfmFiles = files.filter((f) => (0, path_1.extname)(f) === '.usfm');
         }
     }
     if (usfmFiles.length <= 0) {
@@ -79,7 +191,7 @@ async function loadTranslationFiles(translation) {
         }
         const filePath = path.resolve(translation, file);
         promises.push(loadFile(filePath, {
-            translation: metadata
+            translation: metadata,
         }));
     }
     return await Promise.all(promises);
@@ -107,13 +219,15 @@ async function loadTranslationMetadata(translation) {
                 licenseUrl: metadata.copyright.attribution_url,
                 website: metadata.copyright.attribution_url,
                 shortName: metadata.name.abbrev,
-                direction: metadata.direction
+                direction: metadata.direction,
             };
         }
         else {
             const metadataJson = path.resolve(translation, 'metadata.json');
             if ((0, fs_extra_1.existsSync)(metadataJson)) {
-                const data = await (0, promises_1.readFile)(metadataJson, { encoding: 'utf-8' });
+                const data = await (0, promises_1.readFile)(metadataJson, {
+                    encoding: 'utf-8',
+                });
                 return JSON.parse(data);
             }
         }
@@ -129,7 +243,7 @@ async function loadTranslationMetadata(translation) {
 async function loadFile(file, metadata) {
     const extension = path.extname(file);
     const content = await (0, promises_1.readFile)(file, {
-        encoding: 'utf-8'
+        encoding: 'utf-8',
     });
     const hash = (0, hash_js_1.sha256)()
         .update(content)
@@ -230,3 +344,19 @@ function makeRelativePath(path) {
     }
     return path;
 }
+/**
+ * Calculates the SHa256 hash of the given input files.
+ * @param files The files to hash.
+ */
+function hashInputFiles(files) {
+    let sha = (0, hash_js_1.sha256)();
+    for (let file of files) {
+        if (file.sha256) {
+            sha.update(file.sha256);
+        }
+        else {
+            sha.update(file.content);
+        }
+    }
+    return sha.digest('hex');
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@helloao/cli",
-  "version": "0.0.4",
+  "version": "0.0.6",
   "description": "A CLI and related tools for managing HelloAO's Free Bible API",
   "module": "index.js",
   "types": "index.d.ts",
@@ -10,7 +10,7 @@
   "author": "Kallyn Gowdy <kal@helloao.org>",
   "license": "MIT",
   "dependencies": {
-    "@helloao/tools": "^0.0.4",
+    "@helloao/tools": "^0.0.5",
     "commander": "12.1.0",
     "@gracious.tech/fetch-client": "^0.7.0",
     "prisma": "^5.12.1",
@@ -23,7 +23,10 @@
     "hash.js": "1.1.7",
     "@zip.js/zip.js": "^2.6.40",
     "@aws-sdk/client-s3": "^3.609.0",
-    "@aws-sdk/credential-providers": "^3.609.0"
+    "@aws-sdk/credential-providers": "^3.609.0",
+    "@smithy/types": "^3.3.0",
+    "@inquirer/prompts": "5.3.8",
+    "all-iso-language-codes": "1.0.17"
   },
   "files": [
     "/README.md",
@@ -37,7 +40,5 @@
   "prisma": {
     "schema": "./schema.prisma"
   },
-  "scripts": {
-    "postinstall": "prisma generate"
-  }
+  "scripts": {}
 }

package/prisma-gen/index.js

@@ -229,8 +229,8 @@ const fs = require('fs')
 config.dirname = __dirname
 if (!fs.existsSync(path.join(__dirname, 'schema.prisma'))) {
   const alternativePaths = [
-    "prisma-gen",
-    "",
+    "packages/helloao-cli/prisma-gen",
+    "helloao-cli/prisma-gen",
   ]
   
   const alternativePath = alternativePaths.find((altPath) => {
@@ -259,7 +259,7 @@ Object.assign(exports, Prisma)
 
 // file annotations for bundling tools to include these files
 path.join(__dirname, "query_engine-windows.dll.node");
-path.join(process.cwd(), "prisma-gen/query_engine-windows.dll.node")
+path.join(process.cwd(), "packages/helloao-cli/prisma-gen/query_engine-windows.dll.node")
 // file annotations for bundling tools to include these files
 path.join(__dirname, "schema.prisma");
-path.join(process.cwd(), "prisma-gen/schema.prisma")
+path.join(process.cwd(), "packages/helloao-cli/prisma-gen/schema.prisma")

package/s3.d.ts

@@ -1,14 +1,44 @@
-import { SerializedFile, Uploader } from "./db";
+import { SerializedFile, Uploader } from './files';
+import { AwsCredentialIdentity, Provider } from '@smithy/types';
 export declare class S3Uploader implements Uploader {
     private _client;
     private _bucketName;
     private _keyPrefix;
     get idealBatchSize(): number;
-    constructor(bucketName: string, keyPrefix: string, profile: string | null);
+    constructor(bucketName: string, keyPrefix: string, profile: string | null | AwsCredentialIdentity | Provider<AwsCredentialIdentity>);
     upload(file: SerializedFile, overwrite: boolean): Promise<boolean>;
 }
+/**
+ * Parses the given S3 URL into its bucket name and object key.
+ * @param url The URL to parse.
+ */
 export declare function parseS3Url(url: string): {
     bucketName: string;
     objectKey: string;
 } | undefined;
+/**
+ * Gets the HTTP URL for the given S3 URL.
+ * @param s3Url The S3 URL to convert.
+ */
+export declare function getHttpUrl(s3Url: string): string | undefined;
+/**
+ * A provider that gets the credentials directly from the user input.
+ */
+export declare const askForAccessKeyProvider: Provider<AwsCredentialIdentity>;
+/**
+ * Defines a provider that tries to get the credentials from the given list of providers.
+ * @param providers The providers to try.
+ */
+export declare function providerChain(...providers: Provider<AwsCredentialIdentity>[]): Provider<AwsCredentialIdentity>;
+/**
+ * Gets the default provider for the given options.
+ *
+ * Defaults first to using the provided access key and secret access key, then to using the given profile, then finally to asking the user for the access key.
+ * @param options
+ */
+export declare function defaultProviderForOptions(options: {
+    accessKeyId?: string;
+    secretAccessKey?: string;
+    profile?: string;
+}): Provider<AwsCredentialIdentity> | AwsCredentialIdentity;
 //# sourceMappingURL=s3.d.ts.map