@helloao/cli 0.0.5 → 0.0.6

package/schema.prisma

@@ -1,154 +1,154 @@
-datasource db {
-    provider = "sqlite"
-    url      = "file:../../bible-api.dev.db"
-}
-
-generator client {
-    provider = "prisma-client-js"
-    output   = "./prisma-gen"
-}
-
-model Translation {
-    id            String @id
-    name          String
-    website       String
-    licenseUrl    String
-    shortName     String?
-    englishName   String
-    language      String
-    textDirection String
-
-    // The SHA-256 hash of the translation
-    // includes everything about the translation, including the books, chapters, verses, footnotes, etc.
-    sha256         String?
-    
-    books        Book[]
-    chapters     Chapter[]
-    verses       ChapterVerse[]
-    footnotes    ChapterFootnote[]
-    audioUrls    ChapterAudioUrl[]
-}
-
-model InputFile {
-    // The ID of the translation that the file is for
-    translationId String
-
-    // The name of the file
-    name String
-
-    format String
-
-    // The SHA-256 hash of the file
-    sha256 String
-    
-    sizeInBytes Int
-
-    @@id([translationId, name])
-}
-
-model Book {
-    id String
-
-    translationId String
-    translation Translation @relation(fields: [translationId], references: [id])
-
-    name String
-    commonName String
-    title String?
-    order Int
-
-    numberOfChapters Int
-
-    // The SHA-256 hash of the book
-    sha256 String?
-
-    chapters Chapter[]
-    verses ChapterVerse[]
-    footnotes ChapterFootnote[]
-    audioUrls ChapterAudioUrl[]
-
-    @@id([translationId, id])
-}
-
-model Chapter {
-    number Int
-
-    bookId String
-    book Book @relation(fields: [translationId, bookId], references: [translationId, id])
-
-    translationId String
-    translation Translation @relation(fields: [translationId], references: [id])
-
-    json String // The JSON of the chapter
-
-    // The SHA-256 hash of the chapter
-    sha256 String?
-
-    verses ChapterVerse[]
-    footnotes ChapterFootnote[]
-    audioUrls ChapterAudioUrl[]
-
-    @@id([translationId, bookId, number])
-}
-
-model ChapterAudioUrl {
-    number Int
-    bookId String
-    book Book @relation(fields: [translationId, bookId], references: [translationId, id])
-
-    translationId String
-    translation Translation @relation(fields: [translationId], references: [id])
-
-    chapter Chapter @relation(fields: [translationId, bookId, number], references: [translationId, bookId, number])
-
-    reader String
-    url String
-
-    @@id([translationId, bookId, number, reader])
-}
-
-model ChapterVerse {
-    number Int
-
-    chapterNumber Int
-    chapter Chapter @relation(fields: [translationId, bookId, chapterNumber], references: [translationId, bookId, number])
-
-    bookId String
-    book Book @relation(fields: [translationId, bookId], references: [translationId, id])
-
-    translationId String
-    translation Translation @relation(fields: [translationId], references: [id])
-
-    text String // The text of the verse
-    contentJson String // The JSON of the verse content
-
-    // The SHA-256 hash of the verse
-    sha256 String?
-
-    footnotes ChapterFootnote[]
-
-    @@id([translationId, bookId, chapterNumber, number])
-}
-
-model ChapterFootnote {
-    id Int
-
-    chapterNumber Int
-    chapter Chapter @relation(fields: [translationId, bookId, chapterNumber], references: [translationId, bookId, number])
-
-    bookId String
-    book Book @relation(fields: [translationId, bookId], references: [translationId, id])
-
-    translationId String
-    translation Translation @relation(fields: [translationId], references: [id])
-
-    text String
-
-    // The SHA-256 hash of the footnote
-    sha256 String?
-
-    verseNumber Int?
-    verse ChapterVerse? @relation(fields: [translationId, bookId, chapterNumber, verseNumber], references: [translationId, bookId, chapterNumber, number])
-
-    @@id([translationId, bookId, chapterNumber, id])
-}
+datasource db {
+    provider = "sqlite"
+    url      = "file:../../bible-api.dev.db"
+}
+
+generator client {
+    provider = "prisma-client-js"
+    output   = "./prisma-gen"
+}
+
+model Translation {
+    id            String @id
+    name          String
+    website       String
+    licenseUrl    String
+    shortName     String?
+    englishName   String
+    language      String
+    textDirection String
+
+    // The SHA-256 hash of the translation
+    // includes everything about the translation, including the books, chapters, verses, footnotes, etc.
+    sha256         String?
+    
+    books        Book[]
+    chapters     Chapter[]
+    verses       ChapterVerse[]
+    footnotes    ChapterFootnote[]
+    audioUrls    ChapterAudioUrl[]
+}
+
+model InputFile {
+    // The ID of the translation that the file is for
+    translationId String
+
+    // The name of the file
+    name String
+
+    format String
+
+    // The SHA-256 hash of the file
+    sha256 String
+    
+    sizeInBytes Int
+
+    @@id([translationId, name])
+}
+
+model Book {
+    id String
+
+    translationId String
+    translation Translation @relation(fields: [translationId], references: [id])
+
+    name String
+    commonName String
+    title String?
+    order Int
+
+    numberOfChapters Int
+
+    // The SHA-256 hash of the book
+    sha256 String?
+
+    chapters Chapter[]
+    verses ChapterVerse[]
+    footnotes ChapterFootnote[]
+    audioUrls ChapterAudioUrl[]
+
+    @@id([translationId, id])
+}
+
+model Chapter {
+    number Int
+
+    bookId String
+    book Book @relation(fields: [translationId, bookId], references: [translationId, id])
+
+    translationId String
+    translation Translation @relation(fields: [translationId], references: [id])
+
+    json String // The JSON of the chapter
+
+    // The SHA-256 hash of the chapter
+    sha256 String?
+
+    verses ChapterVerse[]
+    footnotes ChapterFootnote[]
+    audioUrls ChapterAudioUrl[]
+
+    @@id([translationId, bookId, number])
+}
+
+model ChapterAudioUrl {
+    number Int
+    bookId String
+    book Book @relation(fields: [translationId, bookId], references: [translationId, id])
+
+    translationId String
+    translation Translation @relation(fields: [translationId], references: [id])
+
+    chapter Chapter @relation(fields: [translationId, bookId, number], references: [translationId, bookId, number])
+
+    reader String
+    url String
+
+    @@id([translationId, bookId, number, reader])
+}
+
+model ChapterVerse {
+    number Int
+
+    chapterNumber Int
+    chapter Chapter @relation(fields: [translationId, bookId, chapterNumber], references: [translationId, bookId, number])
+
+    bookId String
+    book Book @relation(fields: [translationId, bookId], references: [translationId, id])
+
+    translationId String
+    translation Translation @relation(fields: [translationId], references: [id])
+
+    text String // The text of the verse
+    contentJson String // The JSON of the verse content
+
+    // The SHA-256 hash of the verse
+    sha256 String?
+
+    footnotes ChapterFootnote[]
+
+    @@id([translationId, bookId, chapterNumber, number])
+}
+
+model ChapterFootnote {
+    id Int
+
+    chapterNumber Int
+    chapter Chapter @relation(fields: [translationId, bookId, chapterNumber], references: [translationId, bookId, number])
+
+    bookId String
+    book Book @relation(fields: [translationId, bookId], references: [translationId, id])
+
+    translationId String
+    translation Translation @relation(fields: [translationId], references: [id])
+
+    text String
+
+    // The SHA-256 hash of the footnote
+    sha256 String?
+
+    verseNumber Int?
+    verse ChapterVerse? @relation(fields: [translationId, bookId, chapterNumber, verseNumber], references: [translationId, bookId, chapterNumber, number])
+
+    @@id([translationId, bookId, chapterNumber, id])
+}

package/uploads.d.ts

@@ -1,7 +1,9 @@
-import { SerializedFile, Uploader } from "./files";
-import { DatasetOutput } from "@helloao/tools/generation/dataset";
-import { PrismaClient } from "@prisma/client";
-export interface UploadApiFromDatabaseOptions extends UploadApiOptions {
+import { SerializedFile } from './db';
+import { Uploader } from './files';
+import { DatasetOutput } from '@helloao/tools/generation/dataset';
+import { PrismaClient } from './prisma-gen';
+import { GenerateApiOptions } from '@helloao/tools/generation/api';
+export interface UploadApiFromDatabaseOptions extends UploadApiOptions, GenerateApiOptions {
     /**
      * The number of files to upload in each batch.
      */
@@ -29,6 +31,14 @@ export interface UploadApiOptions {
      * The AWS profile to use for uploading to S3.
      */
     profile?: string;
+    /**
+     * The AWS access key ID to use for uploading to S3.
+     */
+    accessKeyId?: string;
+    /**
+     * The AWS secret access key to use for uploading to S3.
+     */
+    secretAccessKey?: string;
     /**
      * Whether to generate API files that use the common name instead of book IDs.
      */
@@ -55,7 +65,7 @@ export declare function uploadApiFilesFromDatabase(db: PrismaClient, dest: strin
  * @param options The options to use for the upload.
  * @param datasets The datasets to generate the API files from.
  */
-export declare function serializeAndUploadDatasets(dest: string, datasets: AsyncIterable<DatasetOutput>, options?: UploadApiOptions): Promise<void>;
+export declare function serializeAndUploadDatasets(dest: string, datasets: AsyncIterable<DatasetOutput>, options?: UploadApiOptions & GenerateApiOptions): Promise<void>;
 /**
  * Uploads the given serialized files to the specified destination.
  * @param dest The destination to upload the API files to. Supported destinations are S3, zip files, and local directories.

package/uploads.js

@@ -34,7 +34,9 @@ async function uploadApiFilesFromDatabase(db, dest, options) {
     if (options.pretty) {
         console.log('Generating pretty-printed JSON files');
     }
-    const pageSize = typeof options.batchSize === 'number' ? options.batchSize : parseInt(options.batchSize);
+    const pageSize = typeof options.batchSize === 'number'
+        ? options.batchSize
+        : parseInt(options.batchSize);
     await serializeAndUploadDatasets(dest, (0, db_1.loadDatasets)(db, pageSize, options.translations), options);
 }
 /**
@@ -66,92 +68,10 @@ async function serializeAndUploadDatasets(dest, datasets, options = {}) {
     if (options.pretty) {
         console.log('Generating pretty-printed JSON files');
     }
-    let uploader;
-    if (dest.startsWith('s3://')) {
-        console.log('Uploading to S3');
-        // Upload to S3
-        const url = dest;
-        const s3Url = (0, s3_1.parseS3Url)(url);
-        if (!s3Url) {
-            throw new Error(`Invalid S3 URL: ${url}`);
-        }
-        if (!s3Url.bucketName) {
-            throw new Error(`Invalid S3 URL: ${url}\nUnable to determine bucket name`);
-        }
-        uploader = new s3_1.S3Uploader(s3Url.bucketName, s3Url.objectKey, options.profile ?? null);
-    }
-    else if (dest.startsWith('console://')) {
-        console.log('Uploading to console');
-        uploader = {
-            idealBatchSize: 50,
-            async upload(file, _overwrite) {
-                console.log(file.path);
-                console.log(file.content);
-                return true;
-            }
-        };
-    }
-    else if ((0, path_1.extname)(dest) === '.zip') {
-        console.log('Writing to zip file:', dest);
-        uploader = new files_1.ZipUploader(dest);
-    }
-    else if (dest) {
-        console.log('Writing to local directory:', dest);
-        uploader = new files_1.FilesUploader(dest);
-    }
-    else {
-        console.error('Unsupported destination:', dest);
-        process.exit(1);
-    }
-    try {
-        for await (let files of (0, db_1.serializeDatasets)(datasets, {
-            useCommonName: !!options.useCommonName,
-            generateAudioFiles: !!options.generateAudioFiles,
-            pretty: !!options.pretty,
-        })) {
-            const batchSize = uploader.idealBatchSize ?? files.length;
-            const totalBatches = Math.ceil(files.length / batchSize);
-            console.log('Uploading', files.length, 'total files');
-            console.log('Uploading in batches of', batchSize);
-            let offset = 0;
-            let batchNumber = 1;
-            let batch = files.slice(offset, offset + batchSize);
-            while (batch.length > 0) {
-                console.log('Uploading batch', batchNumber, 'of', totalBatches);
-                let writtenFiles = 0;
-                const promises = batch.map(async (file) => {
-                    if (filePattern) {
-                        if (!filePattern.test(file.path)) {
-                            console.log('Skipping file:', file.path);
-                            return;
-                        }
-                    }
-                    const isAvailableTranslations = file.path.endsWith('available_translations.json');
-                    const isCommonFile = !isAvailableTranslations;
-                    if (await uploader.upload(file, overwrite || (overwriteCommonFiles && isCommonFile))) {
-                        writtenFiles++;
-                    }
-                    else {
-                        console.warn('File already exists:', file.path);
-                        console.warn('Skipping file');
-                    }
-                    if (file.content instanceof node_stream_1.Readable) {
-                        file.content.destroy();
-                    }
-                });
-                await Promise.all(promises);
-                console.log('Wrote', writtenFiles, 'files');
-                batchNumber++;
-                offset += batchSize;
-                batch = files.slice(offset, offset + batchSize);
-            }
-        }
-    }
-    finally {
-        if (uploader && uploader.dispose) {
-            await uploader.dispose();
-        }
-    }
+    const files = (0, db_1.serializeDatasets)(datasets, {
+        ...options,
+    });
+    await uploadFiles(dest, options, files);
 }
 /**
  * Uploads the given serialized files to the specified destination.
@@ -172,7 +92,7 @@ async function uploadFiles(dest, options, serializedFiles) {
         if (!s3Url.bucketName) {
             throw new Error(`Invalid S3 URL: ${url}\nUnable to determine bucket name`);
         }
-        uploader = new s3_1.S3Uploader(s3Url.bucketName, s3Url.objectKey, options.profile ?? null);
+        uploader = new s3_1.S3Uploader(s3Url.bucketName, s3Url.objectKey, (0, s3_1.defaultProviderForOptions)(options));
     }
     else if (dest.startsWith('console://')) {
         console.log('Uploading to console');
@@ -182,7 +102,7 @@ async function uploadFiles(dest, options, serializedFiles) {
                 console.log(file.path);
                 console.log(file.content);
                 return true;
-            }
+            },
         };
     }
     else if ((0, path_1.extname)(dest) === '.zip') {