@helloao/cli 0.0.4 → 0.0.6

package/s3.js

@@ -1,9 +1,13 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.S3Uploader = void 0;
+exports.askForAccessKeyProvider = exports.S3Uploader = void 0;
 exports.parseS3Url = parseS3Url;
+exports.getHttpUrl = getHttpUrl;
+exports.providerChain = providerChain;
+exports.defaultProviderForOptions = defaultProviderForOptions;
 const client_s3_1 = require("@aws-sdk/client-s3");
 const credential_providers_1 = require("@aws-sdk/credential-providers"); // ES6 import
+const prompts_1 = require("@inquirer/prompts");
 class S3Uploader {
     _client;
     _bucketName;
@@ -15,28 +19,43 @@ class S3Uploader {
         this._bucketName = bucketName;
         this._keyPrefix = keyPrefix;
         this._client = new client_s3_1.S3Client({
-            credentials: (0, credential_providers_1.fromNodeProviderChain)({
-                profile: profile ?? undefined,
-            })
+            credentials: !profile || typeof profile === 'string'
+                ? (0, credential_providers_1.fromNodeProviderChain)({ profile: profile ?? undefined })
+                : profile,
         });
     }
     async upload(file, overwrite) {
-        const path = file.path.startsWith('/') ? file.path.substring(1) : file.path;
+        const path = file.path.startsWith('/')
+            ? file.path.substring(1)
+            : file.path;
         const key = this._keyPrefix ? `${this._keyPrefix}/${path}` : path;
         const hash = file.sha256?.();
         const head = new client_s3_1.HeadObjectCommand({
             Bucket: this._bucketName,
             Key: key,
+            ChecksumMode: 'ENABLED',
         });
         if (hash || !overwrite) {
             try {
                 const existingFile = await this._client.send(head);
-                if (hash && hash.localeCompare(existingFile?.ChecksumSHA256 ?? "", undefined, { sensitivity: 'base' }) === 0) {
-                    // File is already uploaded and matches the checksum.
-                    console.log(`[s3] Matches checksum: ${key}`);
-                    return false;
+                let matches = true;
+                if (hash && existingFile.ChecksumSHA256) {
+                    if (hash.localeCompare(existingFile?.ChecksumSHA256 ?? '', undefined, {
+                        sensitivity: 'base',
+                    }) === 0) {
+                        // File is already uploaded and matches the checksum.
+                        return false;
+                    }
+                    else {
+                        // File is already uploaded but the checksums don't match.
+                        matches = false;
+                    }
+                }
+                else {
+                    // File is already uploaded but the checksum is not available.
+                    console.log(`[s3] Checksum not available: ${key}`);
                 }
-                if (!overwrite) {
+                if (matches && !overwrite) {
                     return false;
                 }
             }
@@ -55,22 +74,96 @@ class S3Uploader {
             Body: file.content,
             ContentType: 'application/json',
             ChecksumSHA256: hash,
+            ChecksumAlgorithm: 'SHA256',
         });
         await this._client.send(command);
         return true;
     }
 }
 exports.S3Uploader = S3Uploader;
+/**
+ * Parses the given S3 URL into its bucket name and object key.
+ * @param url The URL to parse.
+ */
 function parseS3Url(url) {
     const regex = /^s3:\/\/([a-z0-9.\-]+)(\/[^${}]*)?$/;
     const matched = url.match(regex);
     if (matched) {
         const arr = [...matched];
+        let key = arr[2] ?? '';
+        if (key.startsWith('/')) {
+            key = key.substring(1);
+        }
         return {
             bucketName: arr[1],
-            objectKey: arr[2] ?? "",
+            objectKey: key,
         };
     }
     return undefined;
 }
-;
+/**
+ * Gets the HTTP URL for the given S3 URL.
+ * @param s3Url The S3 URL to convert.
+ */
+function getHttpUrl(s3Url) {
+    const parsed = parseS3Url(s3Url);
+    if (!parsed) {
+        return undefined;
+    }
+    const { bucketName, objectKey } = parsed;
+    if (objectKey) {
+        return `https://${bucketName}.s3.amazonaws.com/${objectKey}`;
+    }
+    else {
+        return `https://${bucketName}.s3.amazonaws.com`;
+    }
+}
+/**
+ * A provider that gets the credentials directly from the user input.
+ */
+const askForAccessKeyProvider = async () => {
+    const accessKeyId = await (0, prompts_1.input)({
+        message: 'Enter your AWS Access Key ID',
+    });
+    const secretAccessKey = await (0, prompts_1.password)({
+        message: 'Enter your AWS Secret Access Key',
+    });
+    return {
+        accessKeyId,
+        secretAccessKey,
+    };
+};
+exports.askForAccessKeyProvider = askForAccessKeyProvider;
+/**
+ * Defines a provider that tries to get the credentials from the given list of providers.
+ * @param providers The providers to try.
+ */
+function providerChain(...providers) {
+    return async () => {
+        for (const provider of providers) {
+            const creds = await provider();
+            if (creds?.accessKeyId && creds?.secretAccessKey) {
+                return creds;
+            }
+        }
+        return {
+            accessKeyId: '',
+            secretAccessKey: '',
+        };
+    };
+}
+/**
+ * Gets the default provider for the given options.
+ *
+ * Defaults first to using the provided access key and secret access key, then to using the given profile, then finally to asking the user for the access key.
+ * @param options
+ */
+function defaultProviderForOptions(options) {
+    if (options.accessKeyId && options.secretAccessKey) {
+        return {
+            accessKeyId: options.accessKeyId,
+            secretAccessKey: options.secretAccessKey,
+        };
+    }
+    return providerChain((0, credential_providers_1.fromNodeProviderChain)({ profile: options.profile }), exports.askForAccessKeyProvider);
+}

package/uploads.d.ts

@@ -1,9 +1,15 @@
-import { DatasetOutput } from "@helloao/tools/generation/dataset";
-export interface UploadApiOptions {
+import { SerializedFile } from './db';
+import { Uploader } from './files';
+import { DatasetOutput } from '@helloao/tools/generation/dataset';
+import { PrismaClient } from './prisma-gen';
+import { GenerateApiOptions } from '@helloao/tools/generation/api';
+export interface UploadApiFromDatabaseOptions extends UploadApiOptions, GenerateApiOptions {
     /**
      * The number of files to upload in each batch.
      */
-    batchSize: string;
+    batchSize: string | number;
+}
+export interface UploadApiOptions {
     /**
      * Whether to overwrite existing files.
      */
@@ -25,6 +31,14 @@ export interface UploadApiOptions {
      * The AWS profile to use for uploading to S3.
      */
     profile?: string;
+    /**
+     * The AWS access key ID to use for uploading to S3.
+     */
+    accessKeyId?: string;
+    /**
+     * The AWS secret access key to use for uploading to S3.
+     */
+    secretAccessKey?: string;
     /**
      * Whether to generate API files that use the common name instead of book IDs.
      */
@@ -40,15 +54,30 @@ export interface UploadApiOptions {
 }
 /**
  * Loads and generates the API files from the database and uploads them to the specified destination.
+ * @param db The database that the datasets should be loaded from.
  * @param dest The destination to upload the API files to. Supported destinations are S3, zip files, and local directories.
  * @param options The options to use for the upload.
  */
-export declare function uploadApiFilesFromDatabase(dest: string, options: UploadApiOptions): Promise<void>;
+export declare function uploadApiFilesFromDatabase(db: PrismaClient, dest: string, options: UploadApiFromDatabaseOptions): Promise<void>;
 /**
  * Generates the API files from the given datasets and uploads them to the specified destination.
  * @param dest The destination to upload the API files to. Supported destinations are S3, zip files, and local directories.
  * @param options The options to use for the upload.
  * @param datasets The datasets to generate the API files from.
  */
-export declare function uploadApiFiles(dest: string, options: UploadApiOptions, datasets: AsyncIterable<DatasetOutput>): Promise<void>;
+export declare function serializeAndUploadDatasets(dest: string, datasets: AsyncIterable<DatasetOutput>, options?: UploadApiOptions & GenerateApiOptions): Promise<void>;
+/**
+ * Uploads the given serialized files to the specified destination.
+ * @param dest The destination to upload the API files to. Supported destinations are S3, zip files, and local directories.
+ * @param options The options to use for the upload.
+ * @param datasets The datasets to generate the API files from.
+ */
+export declare function uploadFiles(dest: string, options: UploadApiOptions, serializedFiles: AsyncIterable<SerializedFile[]>): Promise<void>;
+/**
+ * Uploads the given serialized files using the given uploader.
+ * @param uploader The uploader to use.
+ * @param options The options to use for the upload.
+ * @param datasets The datasets to generate the API files from.
+ */
+export declare function uploadFilesUsingUploader(uploader: Uploader, options: UploadApiOptions, serializedFiles: AsyncIterable<SerializedFile[]>): Promise<void>;
 //# sourceMappingURL=uploads.d.ts.map

package/uploads.js

@@ -1,27 +1,43 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.uploadApiFilesFromDatabase = uploadApiFilesFromDatabase;
-exports.uploadApiFiles = uploadApiFiles;
+exports.serializeAndUploadDatasets = serializeAndUploadDatasets;
+exports.uploadFiles = uploadFiles;
+exports.uploadFilesUsingUploader = uploadFilesUsingUploader;
 const db_1 = require("./db");
-const db_2 = require("./db");
 const s3_1 = require("./s3");
 const path_1 = require("path");
 const files_1 = require("./files");
 const node_stream_1 = require("node:stream");
 /**
  * Loads and generates the API files from the database and uploads them to the specified destination.
+ * @param db The database that the datasets should be loaded from.
  * @param dest The destination to upload the API files to. Supported destinations are S3, zip files, and local directories.
  * @param options The options to use for the upload.
  */
-async function uploadApiFilesFromDatabase(dest, options) {
-    const db = (0, db_2.getPrismaDbFromDir)(process.cwd());
-    try {
-        const pageSize = parseInt(options.batchSize);
-        await uploadApiFiles(dest, options, (0, db_1.loadDatasets)(db, pageSize, options.translations));
+async function uploadApiFilesFromDatabase(db, dest, options) {
+    if (options.overwrite) {
+        console.log('Overwriting existing files');
     }
-    finally {
-        db.$disconnect();
+    if (options.overwriteCommonFiles) {
+        console.log('Overwriting only common files');
+    }
+    if (!!options.filePattern) {
+        console.log('Using file pattern:', options.filePattern);
+    }
+    if (options.translations) {
+        console.log('Generating for specific translations:', options.translations);
+    }
+    else {
+        console.log('Generating for all translations');
+    }
+    if (options.pretty) {
+        console.log('Generating pretty-printed JSON files');
     }
+    const pageSize = typeof options.batchSize === 'number'
+        ? options.batchSize
+        : parseInt(options.batchSize);
+    await serializeAndUploadDatasets(dest, (0, db_1.loadDatasets)(db, pageSize, options.translations), options);
 }
 /**
  * Generates the API files from the given datasets and uploads them to the specified destination.
@@ -29,7 +45,7 @@ async function uploadApiFilesFromDatabase(dest, options) {
  * @param options The options to use for the upload.
  * @param datasets The datasets to generate the API files from.
  */
-async function uploadApiFiles(dest, options, datasets) {
+async function serializeAndUploadDatasets(dest, datasets, options = {}) {
     const overwrite = !!options.overwrite;
     if (overwrite) {
         console.log('Overwriting existing files');
@@ -52,6 +68,18 @@ async function uploadApiFiles(dest, options, datasets) {
     if (options.pretty) {
         console.log('Generating pretty-printed JSON files');
     }
+    const files = (0, db_1.serializeDatasets)(datasets, {
+        ...options,
+    });
+    await uploadFiles(dest, options, files);
+}
+/**
+ * Uploads the given serialized files to the specified destination.
+ * @param dest The destination to upload the API files to. Supported destinations are S3, zip files, and local directories.
+ * @param options The options to use for the upload.
+ * @param datasets The datasets to generate the API files from.
+ */
+async function uploadFiles(dest, options, serializedFiles) {
     let uploader;
     if (dest.startsWith('s3://')) {
         console.log('Uploading to S3');
@@ -64,7 +92,7 @@ async function uploadApiFiles(dest, options, datasets) {
         if (!s3Url.bucketName) {
             throw new Error(`Invalid S3 URL: ${url}\nUnable to determine bucket name`);
         }
-        uploader = new s3_1.S3Uploader(s3Url.bucketName, s3Url.objectKey, options.profile ?? null);
+        uploader = new s3_1.S3Uploader(s3Url.bucketName, s3Url.objectKey, (0, s3_1.defaultProviderForOptions)(options));
     }
     else if (dest.startsWith('console://')) {
         console.log('Uploading to console');
@@ -74,7 +102,7 @@ async function uploadApiFiles(dest, options, datasets) {
                 console.log(file.path);
                 console.log(file.content);
                 return true;
-            }
+            },
         };
     }
     else if ((0, path_1.extname)(dest) === '.zip') {
@@ -90,48 +118,7 @@ async function uploadApiFiles(dest, options, datasets) {
         process.exit(1);
     }
     try {
-        for await (let files of (0, db_1.serializeFiles)(datasets, {
-            useCommonName: !!options.useCommonName,
-            generateAudioFiles: !!options.generateAudioFiles,
-            pretty: !!options.pretty,
-        })) {
-            const batchSize = uploader.idealBatchSize ?? files.length;
-            const totalBatches = Math.ceil(files.length / batchSize);
-            console.log('Uploading', files.length, 'total files');
-            console.log('Uploading in batches of', batchSize);
-            let offset = 0;
-            let batchNumber = 1;
-            let batch = files.slice(offset, offset + batchSize);
-            while (batch.length > 0) {
-                console.log('Uploading batch', batchNumber, 'of', totalBatches);
-                let writtenFiles = 0;
-                const promises = batch.map(async (file) => {
-                    if (filePattern) {
-                        if (!filePattern.test(file.path)) {
-                            console.log('Skipping file:', file.path);
-                            return;
-                        }
-                    }
-                    const isAvailableTranslations = file.path.endsWith('available_translations.json');
-                    const isCommonFile = !isAvailableTranslations;
-                    if (await uploader.upload(file, overwrite || (overwriteCommonFiles && isCommonFile))) {
-                        writtenFiles++;
-                    }
-                    else {
-                        console.warn('File already exists:', file.path);
-                        console.warn('Skipping file');
-                    }
-                    if (file.content instanceof node_stream_1.Readable) {
-                        file.content.destroy();
-                    }
-                });
-                await Promise.all(promises);
-                console.log('Wrote', writtenFiles, 'files');
-                batchNumber++;
-                offset += batchSize;
-                batch = files.slice(offset, offset + batchSize);
-            }
-        }
+        await uploadFilesUsingUploader(uploader, options, serializedFiles);
     }
     finally {
         if (uploader && uploader.dispose) {
@@ -139,3 +126,55 @@ async function uploadApiFiles(dest, options, datasets) {
         }
     }
 }
+/**
+ * Uploads the given serialized files using the given uploader.
+ * @param uploader The uploader to use.
+ * @param options The options to use for the upload.
+ * @param datasets The datasets to generate the API files from.
+ */
+async function uploadFilesUsingUploader(uploader, options, serializedFiles) {
+    const overwrite = !!options.overwrite;
+    const overwriteCommonFiles = !!options.overwriteCommonFiles;
+    let filePattern;
+    if (!!options.filePattern) {
+        filePattern = new RegExp(options.filePattern, 'g');
+    }
+    for await (let files of serializedFiles) {
+        const batchSize = uploader.idealBatchSize ?? files.length;
+        const totalBatches = Math.ceil(files.length / batchSize);
+        console.log('Uploading', files.length, 'total files');
+        console.log('Uploading in batches of', batchSize);
+        let offset = 0;
+        let batchNumber = 1;
+        let batch = files.slice(offset, offset + batchSize);
+        while (batch.length > 0) {
+            console.log('Uploading batch', batchNumber, 'of', totalBatches);
+            let writtenFiles = 0;
+            const promises = batch.map(async (file) => {
+                if (filePattern) {
+                    if (!filePattern.test(file.path)) {
+                        console.log('Skipping file:', file.path);
+                        return;
+                    }
+                }
+                const isAvailableTranslations = file.path.endsWith('available_translations.json');
+                const isCommonFile = !isAvailableTranslations;
+                if (await uploader.upload(file, overwrite || (overwriteCommonFiles && isCommonFile))) {
+                    writtenFiles++;
+                }
+                else {
+                    console.warn('File already exists:', file.path);
+                    console.warn('Skipping file');
+                }
+                if (file.content instanceof node_stream_1.Readable) {
+                    file.content.destroy();
+                }
+            });
+            await Promise.all(promises);
+            console.log('Wrote', writtenFiles, 'files');
+            batchNumber++;
+            offset += batchSize;
+            batch = files.slice(offset, offset + batchSize);
+        }
+    }
+}