@helloao/cli 0.0.1

package/s3.js

@@ -0,0 +1,76 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.S3Uploader = void 0;
+exports.parseS3Url = parseS3Url;
+const client_s3_1 = require("@aws-sdk/client-s3");
+const credential_providers_1 = require("@aws-sdk/credential-providers"); // ES6 import
+class S3Uploader {
+    _client;
+    _bucketName;
+    _keyPrefix;
+    get idealBatchSize() {
+        return 50;
+    }
+    constructor(bucketName, keyPrefix, profile) {
+        this._bucketName = bucketName;
+        this._keyPrefix = keyPrefix;
+        this._client = new client_s3_1.S3Client({
+            credentials: (0, credential_providers_1.fromNodeProviderChain)({
+                profile: profile ?? undefined,
+            })
+        });
+    }
+    async upload(file, overwrite) {
+        const path = file.path.startsWith('/') ? file.path.substring(1) : file.path;
+        const key = this._keyPrefix ? `${this._keyPrefix}/${path}` : path;
+        const hash = file.sha256?.();
+        const head = new client_s3_1.HeadObjectCommand({
+            Bucket: this._bucketName,
+            Key: key,
+        });
+        if (hash || !overwrite) {
+            try {
+                const existingFile = await this._client.send(head);
+                if (hash && hash.localeCompare(existingFile?.ChecksumSHA256 ?? "", undefined, { sensitivity: 'base' }) === 0) {
+                    // File is already uploaded and matches the checksum.
+                    console.log(`[s3] Matches checksum: ${key}`);
+                    return false;
+                }
+                if (!overwrite) {
+                    return false;
+                }
+            }
+            catch (err) {
+                if (err instanceof client_s3_1.NotFound) {
+                    // not found, so we can try to write the file.
+                }
+                else {
+                    throw err;
+                }
+            }
+        }
+        const command = new client_s3_1.PutObjectCommand({
+            Bucket: this._bucketName,
+            Key: key,
+            Body: file.content,
+            ContentType: 'application/json',
+            ChecksumSHA256: hash,
+        });
+        await this._client.send(command);
+        return true;
+    }
+}
+exports.S3Uploader = S3Uploader;
+function parseS3Url(url) {
+    const regex = /^s3:\/\/([a-z0-9.\-]+)(\/[^${}]*)?$/;
+    const matched = url.match(regex);
+    if (matched) {
+        const arr = [...matched];
+        return {
+            bucketName: arr[1],
+            objectKey: arr[2] ?? "",
+        };
+    }
+    return undefined;
+}
+;

package/schema.prisma

@@ -0,0 +1,154 @@
+datasource db {
+    provider = "sqlite"
+    url      = "file:../../bible-api.dev.db"
+}
+
+generator client {
+    provider = "prisma-client-js"
+    output   = "./prisma-gen"
+}
+
+model Translation {
+    id            String @id
+    name          String
+    website       String
+    licenseUrl    String
+    shortName     String?
+    englishName   String
+    language      String
+    textDirection String
+
+    // The SHA-256 hash of the translation
+    // includes everything about the translation, including the books, chapters, verses, footnotes, etc.
+    sha256         String?
+    
+    books        Book[]
+    chapters     Chapter[]
+    verses       ChapterVerse[]
+    footnotes    ChapterFootnote[]
+    audioUrls    ChapterAudioUrl[]
+}
+
+model InputFile {
+    // The ID of the translation that the file is for
+    translationId String
+
+    // The name of the file
+    name String
+
+    format String
+
+    // The SHA-256 hash of the file
+    sha256 String
+    
+    sizeInBytes Int
+
+    @@id([translationId, name])
+}
+
+model Book {
+    id String
+
+    translationId String
+    translation Translation @relation(fields: [translationId], references: [id])
+
+    name String
+    commonName String
+    title String?
+    order Int
+
+    numberOfChapters Int
+
+    // The SHA-256 hash of the book
+    sha256 String?
+
+    chapters Chapter[]
+    verses ChapterVerse[]
+    footnotes ChapterFootnote[]
+    audioUrls ChapterAudioUrl[]
+
+    @@id([translationId, id])
+}
+
+model Chapter {
+    number Int
+
+    bookId String
+    book Book @relation(fields: [translationId, bookId], references: [translationId, id])
+
+    translationId String
+    translation Translation @relation(fields: [translationId], references: [id])
+
+    json String // The JSON of the chapter
+
+    // The SHA-256 hash of the chapter
+    sha256 String?
+
+    verses ChapterVerse[]
+    footnotes ChapterFootnote[]
+    audioUrls ChapterAudioUrl[]
+
+    @@id([translationId, bookId, number])
+}
+
+model ChapterAudioUrl {
+    number Int
+    bookId String
+    book Book @relation(fields: [translationId, bookId], references: [translationId, id])
+
+    translationId String
+    translation Translation @relation(fields: [translationId], references: [id])
+
+    chapter Chapter @relation(fields: [translationId, bookId, number], references: [translationId, bookId, number])
+
+    reader String
+    url String
+
+    @@id([translationId, bookId, number, reader])
+}
+
+model ChapterVerse {
+    number Int
+
+    chapterNumber Int
+    chapter Chapter @relation(fields: [translationId, bookId, chapterNumber], references: [translationId, bookId, number])
+
+    bookId String
+    book Book @relation(fields: [translationId, bookId], references: [translationId, id])
+
+    translationId String
+    translation Translation @relation(fields: [translationId], references: [id])
+
+    text String // The text of the verse
+    contentJson String // The JSON of the verse content
+
+    // The SHA-256 hash of the verse
+    sha256 String?
+
+    footnotes ChapterFootnote[]
+
+    @@id([translationId, bookId, chapterNumber, number])
+}
+
+model ChapterFootnote {
+    id Int
+
+    chapterNumber Int
+    chapter Chapter @relation(fields: [translationId, bookId, chapterNumber], references: [translationId, bookId, number])
+
+    bookId String
+    book Book @relation(fields: [translationId, bookId], references: [translationId, id])
+
+    translationId String
+    translation Translation @relation(fields: [translationId], references: [id])
+
+    text String
+
+    // The SHA-256 hash of the footnote
+    sha256 String?
+
+    verseNumber Int?
+    verse ChapterVerse? @relation(fields: [translationId, bookId, chapterNumber, verseNumber], references: [translationId, bookId, chapterNumber, number])
+
+    @@id([translationId, bookId, chapterNumber, id])
+}

package/uploads.d.ts

@@ -0,0 +1,54 @@
+import { DatasetOutput } from "@helloao/tools/generation/dataset";
+export interface UploadApiOptions {
+    /**
+     * The number of files to upload in each batch.
+     */
+    batchSize: string;
+    /**
+     * Whether to overwrite existing files.
+     */
+    overwrite?: boolean;
+    /**
+     * Whether to only overwrite common files.
+     * "Common files" are files that are similar between translations, like the books.json endpoint, or individual chapter endpoints.
+     */
+    overwriteCommonFiles?: boolean;
+    /**
+     * The file pattern regex that should be used to filter the files that are uploaded.
+     */
+    filePattern?: string;
+    /**
+     * The translations to generate API files for.
+     */
+    translations?: string[];
+    /**
+     * The AWS profile to use for uploading to S3.
+     */
+    profile?: string;
+    /**
+     * Whether to generate API files that use the common name instead of book IDs.
+     */
+    useCommonName?: boolean;
+    /**
+     * Whether to generate audio files for the API.
+     */
+    generateAudioFiles?: boolean;
+    /**
+     * Whether to generate pretty-printed JSON files.
+     */
+    pretty?: boolean;
+}
+/**
+ * Loads and generates the API files from the database and uploads them to the specified destination.
+ * @param dest The destination to upload the API files to. Supported destinations are S3, zip files, and local directories.
+ * @param options The options to use for the upload.
+ */
+export declare function uploadApiFilesFromDatabase(dest: string, options: UploadApiOptions): Promise<void>;
+/**
+ * Generates the API files from the given datasets and uploads them to the specified destination.
+ * @param dest The destination to upload the API files to. Supported destinations are S3, zip files, and local directories.
+ * @param options The options to use for the upload.
+ * @param datasets The datasets to generate the API files from.
+ */
+export declare function uploadApiFiles(dest: string, options: UploadApiOptions, datasets: AsyncIterable<DatasetOutput>): Promise<void>;
+//# sourceMappingURL=uploads.d.ts.map

package/uploads.js

@@ -0,0 +1,141 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.uploadApiFilesFromDatabase = uploadApiFilesFromDatabase;
+exports.uploadApiFiles = uploadApiFiles;
+const db_1 = require("./db");
+const db_2 = require("./db");
+const s3_1 = require("./s3");
+const path_1 = require("path");
+const files_1 = require("./files");
+const node_stream_1 = require("node:stream");
+/**
+ * Loads and generates the API files from the database and uploads them to the specified destination.
+ * @param dest The destination to upload the API files to. Supported destinations are S3, zip files, and local directories.
+ * @param options The options to use for the upload.
+ */
+async function uploadApiFilesFromDatabase(dest, options) {
+    const db = (0, db_2.getPrismaDbFromDir)(process.cwd());
+    try {
+        const pageSize = parseInt(options.batchSize);
+        await uploadApiFiles(dest, options, (0, db_1.loadDatasets)(db, pageSize, options.translations));
+    }
+    finally {
+        db.$disconnect();
+    }
+}
+/**
+ * Generates the API files from the given datasets and uploads them to the specified destination.
+ * @param dest The destination to upload the API files to. Supported destinations are S3, zip files, and local directories.
+ * @param options The options to use for the upload.
+ * @param datasets The datasets to generate the API files from.
+ */
+async function uploadApiFiles(dest, options, datasets) {
+    const overwrite = !!options.overwrite;
+    if (overwrite) {
+        console.log('Overwriting existing files');
+    }
+    const overwriteCommonFiles = !!options.overwriteCommonFiles;
+    if (overwriteCommonFiles) {
+        console.log('Overwriting only common files');
+    }
+    let filePattern;
+    if (!!options.filePattern) {
+        filePattern = new RegExp(options.filePattern, 'g');
+        console.log('Using file pattern:', filePattern);
+    }
+    if (options.translations) {
+        console.log('Generating for specific translations:', options.translations);
+    }
+    else {
+        console.log('Generating for all translations');
+    }
+    if (options.pretty) {
+        console.log('Generating pretty-printed JSON files');
+    }
+    let uploader;
+    if (dest.startsWith('s3://')) {
+        console.log('Uploading to S3');
+        // Upload to S3
+        const url = dest;
+        const s3Url = (0, s3_1.parseS3Url)(url);
+        if (!s3Url) {
+            throw new Error(`Invalid S3 URL: ${url}`);
+        }
+        if (!s3Url.bucketName) {
+            throw new Error(`Invalid S3 URL: ${url}\nUnable to determine bucket name`);
+        }
+        uploader = new s3_1.S3Uploader(s3Url.bucketName, s3Url.objectKey, options.profile ?? null);
+    }
+    else if (dest.startsWith('console://')) {
+        console.log('Uploading to console');
+        uploader = {
+            idealBatchSize: 50,
+            async upload(file, _overwrite) {
+                console.log(file.path);
+                console.log(file.content);
+                return true;
+            }
+        };
+    }
+    else if ((0, path_1.extname)(dest) === '.zip') {
+        console.log('Writing to zip file:', dest);
+        uploader = new files_1.ZipUploader(dest);
+    }
+    else if (dest) {
+        console.log('Writing to local directory:', dest);
+        uploader = new files_1.FilesUploader(dest);
+    }
+    else {
+        console.error('Unsupported destination:', dest);
+        process.exit(1);
+    }
+    try {
+        for await (let files of (0, db_1.serializeFiles)(datasets, {
+            useCommonName: !!options.useCommonName,
+            generateAudioFiles: !!options.generateAudioFiles,
+            pretty: !!options.pretty,
+        })) {
+            const batchSize = uploader.idealBatchSize ?? files.length;
+            const totalBatches = Math.ceil(files.length / batchSize);
+            console.log('Uploading', files.length, 'total files');
+            console.log('Uploading in batches of', batchSize);
+            let offset = 0;
+            let batchNumber = 1;
+            let batch = files.slice(offset, offset + batchSize);
+            while (batch.length > 0) {
+                console.log('Uploading batch', batchNumber, 'of', totalBatches);
+                let writtenFiles = 0;
+                const promises = batch.map(async (file) => {
+                    if (filePattern) {
+                        if (!filePattern.test(file.path)) {
+                            console.log('Skipping file:', file.path);
+                            return;
+                        }
+                    }
+                    const isAvailableTranslations = file.path.endsWith('available_translations.json');
+                    const isCommonFile = !isAvailableTranslations;
+                    if (await uploader.upload(file, overwrite || (overwriteCommonFiles && isCommonFile))) {
+                        writtenFiles++;
+                    }
+                    else {
+                        console.warn('File already exists:', file.path);
+                        console.warn('Skipping file');
+                    }
+                    if (file.content instanceof node_stream_1.Readable) {
+                        file.content.destroy();
+                    }
+                });
+                await Promise.all(promises);
+                console.log('Wrote', writtenFiles, 'files');
+                batchNumber++;
+                offset += batchSize;
+                batch = files.slice(offset, offset + batchSize);
+            }
+        }
+    }
+    finally {
+        if (uploader && uploader.dispose) {
+            await uploader.dispose();
+        }
+    }
+}