@helloao/cli 0.0.4 → 0.0.5

package/schema.prisma

@@ -1,154 +1,154 @@
-datasource db {
-    provider = "sqlite"
-    url      = "file:../../bible-api.dev.db"
-}
-
-generator client {
-    provider = "prisma-client-js"
-    output   = "./prisma-gen"
-}
-
-model Translation {
-    id            String @id
-    name          String
-    website       String
-    licenseUrl    String
-    shortName     String?
-    englishName   String
-    language      String
-    textDirection String
-
-    // The SHA-256 hash of the translation
-    // includes everything about the translation, including the books, chapters, verses, footnotes, etc.
-    sha256         String?
-    
-    books        Book[]
-    chapters     Chapter[]
-    verses       ChapterVerse[]
-    footnotes    ChapterFootnote[]
-    audioUrls    ChapterAudioUrl[]
-}
-
-model InputFile {
-    // The ID of the translation that the file is for
-    translationId String
-
-    // The name of the file
-    name String
-
-    format String
-
-    // The SHA-256 hash of the file
-    sha256 String
-    
-    sizeInBytes Int
-
-    @@id([translationId, name])
-}
-
-model Book {
-    id String
-
-    translationId String
-    translation Translation @relation(fields: [translationId], references: [id])
-
-    name String
-    commonName String
-    title String?
-    order Int
-
-    numberOfChapters Int
-
-    // The SHA-256 hash of the book
-    sha256 String?
-
-    chapters Chapter[]
-    verses ChapterVerse[]
-    footnotes ChapterFootnote[]
-    audioUrls ChapterAudioUrl[]
-
-    @@id([translationId, id])
-}
-
-model Chapter {
-    number Int
-
-    bookId String
-    book Book @relation(fields: [translationId, bookId], references: [translationId, id])
-
-    translationId String
-    translation Translation @relation(fields: [translationId], references: [id])
-
-    json String // The JSON of the chapter
-
-    // The SHA-256 hash of the chapter
-    sha256 String?
-
-    verses ChapterVerse[]
-    footnotes ChapterFootnote[]
-    audioUrls ChapterAudioUrl[]
-
-    @@id([translationId, bookId, number])
-}
-
-model ChapterAudioUrl {
-    number Int
-    bookId String
-    book Book @relation(fields: [translationId, bookId], references: [translationId, id])
-
-    translationId String
-    translation Translation @relation(fields: [translationId], references: [id])
-
-    chapter Chapter @relation(fields: [translationId, bookId, number], references: [translationId, bookId, number])
-
-    reader String
-    url String
-
-    @@id([translationId, bookId, number, reader])
-}
-
-model ChapterVerse {
-    number Int
-
-    chapterNumber Int
-    chapter Chapter @relation(fields: [translationId, bookId, chapterNumber], references: [translationId, bookId, number])
-
-    bookId String
-    book Book @relation(fields: [translationId, bookId], references: [translationId, id])
-
-    translationId String
-    translation Translation @relation(fields: [translationId], references: [id])
-
-    text String // The text of the verse
-    contentJson String // The JSON of the verse content
-
-    // The SHA-256 hash of the verse
-    sha256 String?
-
-    footnotes ChapterFootnote[]
-
-    @@id([translationId, bookId, chapterNumber, number])
-}
-
-model ChapterFootnote {
-    id Int
-
-    chapterNumber Int
-    chapter Chapter @relation(fields: [translationId, bookId, chapterNumber], references: [translationId, bookId, number])
-
-    bookId String
-    book Book @relation(fields: [translationId, bookId], references: [translationId, id])
-
-    translationId String
-    translation Translation @relation(fields: [translationId], references: [id])
-
-    text String
-
-    // The SHA-256 hash of the footnote
-    sha256 String?
-
-    verseNumber Int?
-    verse ChapterVerse? @relation(fields: [translationId, bookId, chapterNumber, verseNumber], references: [translationId, bookId, chapterNumber, number])
-
-    @@id([translationId, bookId, chapterNumber, id])
-}
+datasource db {
+    provider = "sqlite"
+    url      = "file:../../bible-api.dev.db"
+}
+
+generator client {
+    provider = "prisma-client-js"
+    output   = "./prisma-gen"
+}
+
+model Translation {
+    id            String @id
+    name          String
+    website       String
+    licenseUrl    String
+    shortName     String?
+    englishName   String
+    language      String
+    textDirection String
+
+    // The SHA-256 hash of the translation
+    // includes everything about the translation, including the books, chapters, verses, footnotes, etc.
+    sha256         String?
+    
+    books        Book[]
+    chapters     Chapter[]
+    verses       ChapterVerse[]
+    footnotes    ChapterFootnote[]
+    audioUrls    ChapterAudioUrl[]
+}
+
+model InputFile {
+    // The ID of the translation that the file is for
+    translationId String
+
+    // The name of the file
+    name String
+
+    format String
+
+    // The SHA-256 hash of the file
+    sha256 String
+    
+    sizeInBytes Int
+
+    @@id([translationId, name])
+}
+
+model Book {
+    id String
+
+    translationId String
+    translation Translation @relation(fields: [translationId], references: [id])
+
+    name String
+    commonName String
+    title String?
+    order Int
+
+    numberOfChapters Int
+
+    // The SHA-256 hash of the book
+    sha256 String?
+
+    chapters Chapter[]
+    verses ChapterVerse[]
+    footnotes ChapterFootnote[]
+    audioUrls ChapterAudioUrl[]
+
+    @@id([translationId, id])
+}
+
+model Chapter {
+    number Int
+
+    bookId String
+    book Book @relation(fields: [translationId, bookId], references: [translationId, id])
+
+    translationId String
+    translation Translation @relation(fields: [translationId], references: [id])
+
+    json String // The JSON of the chapter
+
+    // The SHA-256 hash of the chapter
+    sha256 String?
+
+    verses ChapterVerse[]
+    footnotes ChapterFootnote[]
+    audioUrls ChapterAudioUrl[]
+
+    @@id([translationId, bookId, number])
+}
+
+model ChapterAudioUrl {
+    number Int
+    bookId String
+    book Book @relation(fields: [translationId, bookId], references: [translationId, id])
+
+    translationId String
+    translation Translation @relation(fields: [translationId], references: [id])
+
+    chapter Chapter @relation(fields: [translationId, bookId, number], references: [translationId, bookId, number])
+
+    reader String
+    url String
+
+    @@id([translationId, bookId, number, reader])
+}
+
+model ChapterVerse {
+    number Int
+
+    chapterNumber Int
+    chapter Chapter @relation(fields: [translationId, bookId, chapterNumber], references: [translationId, bookId, number])
+
+    bookId String
+    book Book @relation(fields: [translationId, bookId], references: [translationId, id])
+
+    translationId String
+    translation Translation @relation(fields: [translationId], references: [id])
+
+    text String // The text of the verse
+    contentJson String // The JSON of the verse content
+
+    // The SHA-256 hash of the verse
+    sha256 String?
+
+    footnotes ChapterFootnote[]
+
+    @@id([translationId, bookId, chapterNumber, number])
+}
+
+model ChapterFootnote {
+    id Int
+
+    chapterNumber Int
+    chapter Chapter @relation(fields: [translationId, bookId, chapterNumber], references: [translationId, bookId, number])
+
+    bookId String
+    book Book @relation(fields: [translationId, bookId], references: [translationId, id])
+
+    translationId String
+    translation Translation @relation(fields: [translationId], references: [id])
+
+    text String
+
+    // The SHA-256 hash of the footnote
+    sha256 String?
+
+    verseNumber Int?
+    verse ChapterVerse? @relation(fields: [translationId, bookId, chapterNumber, verseNumber], references: [translationId, bookId, chapterNumber, number])
+
+    @@id([translationId, bookId, chapterNumber, id])
+}

package/uploads.d.ts

@@ -1,9 +1,13 @@
+import { SerializedFile, Uploader } from "./files";
 import { DatasetOutput } from "@helloao/tools/generation/dataset";
-export interface UploadApiOptions {
+import { PrismaClient } from "@prisma/client";
+export interface UploadApiFromDatabaseOptions extends UploadApiOptions {
     /**
      * The number of files to upload in each batch.
      */
-    batchSize: string;
+    batchSize: string | number;
+}
+export interface UploadApiOptions {
     /**
      * Whether to overwrite existing files.
      */
@@ -40,15 +44,30 @@ export interface UploadApiOptions {
 }
 /**
  * Loads and generates the API files from the database and uploads them to the specified destination.
+ * @param db The database that the datasets should be loaded from.
  * @param dest The destination to upload the API files to. Supported destinations are S3, zip files, and local directories.
  * @param options The options to use for the upload.
  */
-export declare function uploadApiFilesFromDatabase(dest: string, options: UploadApiOptions): Promise<void>;
+export declare function uploadApiFilesFromDatabase(db: PrismaClient, dest: string, options: UploadApiFromDatabaseOptions): Promise<void>;
 /**
  * Generates the API files from the given datasets and uploads them to the specified destination.
  * @param dest The destination to upload the API files to. Supported destinations are S3, zip files, and local directories.
  * @param options The options to use for the upload.
  * @param datasets The datasets to generate the API files from.
  */
-export declare function uploadApiFiles(dest: string, options: UploadApiOptions, datasets: AsyncIterable<DatasetOutput>): Promise<void>;
+export declare function serializeAndUploadDatasets(dest: string, datasets: AsyncIterable<DatasetOutput>, options?: UploadApiOptions): Promise<void>;
+/**
+ * Uploads the given serialized files to the specified destination.
+ * @param dest The destination to upload the API files to. Supported destinations are S3, zip files, and local directories.
+ * @param options The options to use for the upload.
+ * @param datasets The datasets to generate the API files from.
+ */
+export declare function uploadFiles(dest: string, options: UploadApiOptions, serializedFiles: AsyncIterable<SerializedFile[]>): Promise<void>;
+/**
+ * Uploads the given serialized files using the given uploader.
+ * @param uploader The uploader to use.
+ * @param options The options to use for the upload.
+ * @param datasets The datasets to generate the API files from.
+ */
+export declare function uploadFilesUsingUploader(uploader: Uploader, options: UploadApiOptions, serializedFiles: AsyncIterable<SerializedFile[]>): Promise<void>;
 //# sourceMappingURL=uploads.d.ts.map

package/uploads.js

@@ -1,27 +1,41 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.uploadApiFilesFromDatabase = uploadApiFilesFromDatabase;
-exports.uploadApiFiles = uploadApiFiles;
+exports.serializeAndUploadDatasets = serializeAndUploadDatasets;
+exports.uploadFiles = uploadFiles;
+exports.uploadFilesUsingUploader = uploadFilesUsingUploader;
 const db_1 = require("./db");
-const db_2 = require("./db");
 const s3_1 = require("./s3");
 const path_1 = require("path");
 const files_1 = require("./files");
 const node_stream_1 = require("node:stream");
 /**
  * Loads and generates the API files from the database and uploads them to the specified destination.
+ * @param db The database that the datasets should be loaded from.
  * @param dest The destination to upload the API files to. Supported destinations are S3, zip files, and local directories.
  * @param options The options to use for the upload.
  */
-async function uploadApiFilesFromDatabase(dest, options) {
-    const db = (0, db_2.getPrismaDbFromDir)(process.cwd());
-    try {
-        const pageSize = parseInt(options.batchSize);
-        await uploadApiFiles(dest, options, (0, db_1.loadDatasets)(db, pageSize, options.translations));
+async function uploadApiFilesFromDatabase(db, dest, options) {
+    if (options.overwrite) {
+        console.log('Overwriting existing files');
     }
-    finally {
-        db.$disconnect();
+    if (options.overwriteCommonFiles) {
+        console.log('Overwriting only common files');
+    }
+    if (!!options.filePattern) {
+        console.log('Using file pattern:', options.filePattern);
+    }
+    if (options.translations) {
+        console.log('Generating for specific translations:', options.translations);
+    }
+    else {
+        console.log('Generating for all translations');
+    }
+    if (options.pretty) {
+        console.log('Generating pretty-printed JSON files');
     }
+    const pageSize = typeof options.batchSize === 'number' ? options.batchSize : parseInt(options.batchSize);
+    await serializeAndUploadDatasets(dest, (0, db_1.loadDatasets)(db, pageSize, options.translations), options);
 }
 /**
  * Generates the API files from the given datasets and uploads them to the specified destination.
@@ -29,7 +43,7 @@ async function uploadApiFilesFromDatabase(dest, options) {
  * @param options The options to use for the upload.
  * @param datasets The datasets to generate the API files from.
  */
-async function uploadApiFiles(dest, options, datasets) {
+async function serializeAndUploadDatasets(dest, datasets, options = {}) {
     const overwrite = !!options.overwrite;
     if (overwrite) {
         console.log('Overwriting existing files');
@@ -90,7 +104,7 @@ async function uploadApiFiles(dest, options, datasets) {
         process.exit(1);
     }
     try {
-        for await (let files of (0, db_1.serializeFiles)(datasets, {
+        for await (let files of (0, db_1.serializeDatasets)(datasets, {
             useCommonName: !!options.useCommonName,
             generateAudioFiles: !!options.generateAudioFiles,
             pretty: !!options.pretty,
@@ -139,3 +153,108 @@ async function uploadApiFiles(dest, options, datasets) {
         }
     }
 }
+/**
+ * Uploads the given serialized files to the specified destination.
+ * @param dest The destination to upload the API files to. Supported destinations are S3, zip files, and local directories.
+ * @param options The options to use for the upload.
+ * @param datasets The datasets to generate the API files from.
+ */
+async function uploadFiles(dest, options, serializedFiles) {
+    let uploader;
+    if (dest.startsWith('s3://')) {
+        console.log('Uploading to S3');
+        // Upload to S3
+        const url = dest;
+        const s3Url = (0, s3_1.parseS3Url)(url);
+        if (!s3Url) {
+            throw new Error(`Invalid S3 URL: ${url}`);
+        }
+        if (!s3Url.bucketName) {
+            throw new Error(`Invalid S3 URL: ${url}\nUnable to determine bucket name`);
+        }
+        uploader = new s3_1.S3Uploader(s3Url.bucketName, s3Url.objectKey, options.profile ?? null);
+    }
+    else if (dest.startsWith('console://')) {
+        console.log('Uploading to console');
+        uploader = {
+            idealBatchSize: 50,
+            async upload(file, _overwrite) {
+                console.log(file.path);
+                console.log(file.content);
+                return true;
+            }
+        };
+    }
+    else if ((0, path_1.extname)(dest) === '.zip') {
+        console.log('Writing to zip file:', dest);
+        uploader = new files_1.ZipUploader(dest);
+    }
+    else if (dest) {
+        console.log('Writing to local directory:', dest);
+        uploader = new files_1.FilesUploader(dest);
+    }
+    else {
+        console.error('Unsupported destination:', dest);
+        process.exit(1);
+    }
+    try {
+        await uploadFilesUsingUploader(uploader, options, serializedFiles);
+    }
+    finally {
+        if (uploader && uploader.dispose) {
+            await uploader.dispose();
+        }
+    }
+}
+/**
+ * Uploads the given serialized files using the given uploader.
+ * @param uploader The uploader to use.
+ * @param options The options to use for the upload.
+ * @param datasets The datasets to generate the API files from.
+ */
+async function uploadFilesUsingUploader(uploader, options, serializedFiles) {
+    const overwrite = !!options.overwrite;
+    const overwriteCommonFiles = !!options.overwriteCommonFiles;
+    let filePattern;
+    if (!!options.filePattern) {
+        filePattern = new RegExp(options.filePattern, 'g');
+    }
+    for await (let files of serializedFiles) {
+        const batchSize = uploader.idealBatchSize ?? files.length;
+        const totalBatches = Math.ceil(files.length / batchSize);
+        console.log('Uploading', files.length, 'total files');
+        console.log('Uploading in batches of', batchSize);
+        let offset = 0;
+        let batchNumber = 1;
+        let batch = files.slice(offset, offset + batchSize);
+        while (batch.length > 0) {
+            console.log('Uploading batch', batchNumber, 'of', totalBatches);
+            let writtenFiles = 0;
+            const promises = batch.map(async (file) => {
+                if (filePattern) {
+                    if (!filePattern.test(file.path)) {
+                        console.log('Skipping file:', file.path);
+                        return;
+                    }
+                }
+                const isAvailableTranslations = file.path.endsWith('available_translations.json');
+                const isCommonFile = !isAvailableTranslations;
+                if (await uploader.upload(file, overwrite || (overwriteCommonFiles && isCommonFile))) {
+                    writtenFiles++;
+                }
+                else {
+                    console.warn('File already exists:', file.path);
+                    console.warn('Skipping file');
+                }
+                if (file.content instanceof node_stream_1.Readable) {
+                    file.content.destroy();
+                }
+            });
+            await Promise.all(promises);
+            console.log('Wrote', writtenFiles, 'files');
+            batchNumber++;
+            offset += batchSize;
+            batch = files.slice(offset, offset + batchSize);
+        }
+    }
+}