@helloao/cli 0.0.6 → 0.0.7

package/s3.js

@@ -1,169 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.askForAccessKeyProvider = exports.S3Uploader = void 0;
-exports.parseS3Url = parseS3Url;
-exports.getHttpUrl = getHttpUrl;
-exports.providerChain = providerChain;
-exports.defaultProviderForOptions = defaultProviderForOptions;
-const client_s3_1 = require("@aws-sdk/client-s3");
-const credential_providers_1 = require("@aws-sdk/credential-providers"); // ES6 import
-const prompts_1 = require("@inquirer/prompts");
-class S3Uploader {
-    _client;
-    _bucketName;
-    _keyPrefix;
-    get idealBatchSize() {
-        return 50;
-    }
-    constructor(bucketName, keyPrefix, profile) {
-        this._bucketName = bucketName;
-        this._keyPrefix = keyPrefix;
-        this._client = new client_s3_1.S3Client({
-            credentials: !profile || typeof profile === 'string'
-                ? (0, credential_providers_1.fromNodeProviderChain)({ profile: profile ?? undefined })
-                : profile,
-        });
-    }
-    async upload(file, overwrite) {
-        const path = file.path.startsWith('/')
-            ? file.path.substring(1)
-            : file.path;
-        const key = this._keyPrefix ? `${this._keyPrefix}/${path}` : path;
-        const hash = file.sha256?.();
-        const head = new client_s3_1.HeadObjectCommand({
-            Bucket: this._bucketName,
-            Key: key,
-            ChecksumMode: 'ENABLED',
-        });
-        if (hash || !overwrite) {
-            try {
-                const existingFile = await this._client.send(head);
-                let matches = true;
-                if (hash && existingFile.ChecksumSHA256) {
-                    if (hash.localeCompare(existingFile?.ChecksumSHA256 ?? '', undefined, {
-                        sensitivity: 'base',
-                    }) === 0) {
-                        // File is already uploaded and matches the checksum.
-                        return false;
-                    }
-                    else {
-                        // File is already uploaded but the checksums don't match.
-                        matches = false;
-                    }
-                }
-                else {
-                    // File is already uploaded but the checksum is not available.
-                    console.log(`[s3] Checksum not available: ${key}`);
-                }
-                if (matches && !overwrite) {
-                    return false;
-                }
-            }
-            catch (err) {
-                if (err instanceof client_s3_1.NotFound) {
-                    // not found, so we can try to write the file.
-                }
-                else {
-                    throw err;
-                }
-            }
-        }
-        const command = new client_s3_1.PutObjectCommand({
-            Bucket: this._bucketName,
-            Key: key,
-            Body: file.content,
-            ContentType: 'application/json',
-            ChecksumSHA256: hash,
-            ChecksumAlgorithm: 'SHA256',
-        });
-        await this._client.send(command);
-        return true;
-    }
-}
-exports.S3Uploader = S3Uploader;
-/**
- * Parses the given S3 URL into its bucket name and object key.
- * @param url The URL to parse.
- */
-function parseS3Url(url) {
-    const regex = /^s3:\/\/([a-z0-9.\-]+)(\/[^${}]*)?$/;
-    const matched = url.match(regex);
-    if (matched) {
-        const arr = [...matched];
-        let key = arr[2] ?? '';
-        if (key.startsWith('/')) {
-            key = key.substring(1);
-        }
-        return {
-            bucketName: arr[1],
-            objectKey: key,
-        };
-    }
-    return undefined;
-}
-/**
- * Gets the HTTP URL for the given S3 URL.
- * @param s3Url The S3 URL to convert.
- */
-function getHttpUrl(s3Url) {
-    const parsed = parseS3Url(s3Url);
-    if (!parsed) {
-        return undefined;
-    }
-    const { bucketName, objectKey } = parsed;
-    if (objectKey) {
-        return `https://${bucketName}.s3.amazonaws.com/${objectKey}`;
-    }
-    else {
-        return `https://${bucketName}.s3.amazonaws.com`;
-    }
-}
-/**
- * A provider that gets the credentials directly from the user input.
- */
-const askForAccessKeyProvider = async () => {
-    const accessKeyId = await (0, prompts_1.input)({
-        message: 'Enter your AWS Access Key ID',
-    });
-    const secretAccessKey = await (0, prompts_1.password)({
-        message: 'Enter your AWS Secret Access Key',
-    });
-    return {
-        accessKeyId,
-        secretAccessKey,
-    };
-};
-exports.askForAccessKeyProvider = askForAccessKeyProvider;
-/**
- * Defines a provider that tries to get the credentials from the given list of providers.
- * @param providers The providers to try.
- */
-function providerChain(...providers) {
-    return async () => {
-        for (const provider of providers) {
-            const creds = await provider();
-            if (creds?.accessKeyId && creds?.secretAccessKey) {
-                return creds;
-            }
-        }
-        return {
-            accessKeyId: '',
-            secretAccessKey: '',
-        };
-    };
-}
-/**
- * Gets the default provider for the given options.
- *
- * Defaults first to using the provided access key and secret access key, then to using the given profile, then finally to asking the user for the access key.
- * @param options
- */
-function defaultProviderForOptions(options) {
-    if (options.accessKeyId && options.secretAccessKey) {
-        return {
-            accessKeyId: options.accessKeyId,
-            secretAccessKey: options.secretAccessKey,
-        };
-    }
-    return providerChain((0, credential_providers_1.fromNodeProviderChain)({ profile: options.profile }), exports.askForAccessKeyProvider);
-}

package/uploads.d.ts

@@ -1,83 +0,0 @@
-import { SerializedFile } from './db';
-import { Uploader } from './files';
-import { DatasetOutput } from '@helloao/tools/generation/dataset';
-import { PrismaClient } from './prisma-gen';
-import { GenerateApiOptions } from '@helloao/tools/generation/api';
-export interface UploadApiFromDatabaseOptions extends UploadApiOptions, GenerateApiOptions {
-    /**
-     * The number of files to upload in each batch.
-     */
-    batchSize: string | number;
-}
-export interface UploadApiOptions {
-    /**
-     * Whether to overwrite existing files.
-     */
-    overwrite?: boolean;
-    /**
-     * Whether to only overwrite common files.
-     * "Common files" are files that are similar between translations, like the books.json endpoint, or individual chapter endpoints.
-     */
-    overwriteCommonFiles?: boolean;
-    /**
-     * The file pattern regex that should be used to filter the files that are uploaded.
-     */
-    filePattern?: string;
-    /**
-     * The translations to generate API files for.
-     */
-    translations?: string[];
-    /**
-     * The AWS profile to use for uploading to S3.
-     */
-    profile?: string;
-    /**
-     * The AWS access key ID to use for uploading to S3.
-     */
-    accessKeyId?: string;
-    /**
-     * The AWS secret access key to use for uploading to S3.
-     */
-    secretAccessKey?: string;
-    /**
-     * Whether to generate API files that use the common name instead of book IDs.
-     */
-    useCommonName?: boolean;
-    /**
-     * Whether to generate audio files for the API.
-     */
-    generateAudioFiles?: boolean;
-    /**
-     * Whether to generate pretty-printed JSON files.
-     */
-    pretty?: boolean;
-}
-/**
- * Loads and generates the API files from the database and uploads them to the specified destination.
- * @param db The database that the datasets should be loaded from.
- * @param dest The destination to upload the API files to. Supported destinations are S3, zip files, and local directories.
- * @param options The options to use for the upload.
- */
-export declare function uploadApiFilesFromDatabase(db: PrismaClient, dest: string, options: UploadApiFromDatabaseOptions): Promise<void>;
-/**
- * Generates the API files from the given datasets and uploads them to the specified destination.
- * @param dest The destination to upload the API files to. Supported destinations are S3, zip files, and local directories.
- * @param options The options to use for the upload.
- * @param datasets The datasets to generate the API files from.
- */
-export declare function serializeAndUploadDatasets(dest: string, datasets: AsyncIterable<DatasetOutput>, options?: UploadApiOptions & GenerateApiOptions): Promise<void>;
-/**
- * Uploads the given serialized files to the specified destination.
- * @param dest The destination to upload the API files to. Supported destinations are S3, zip files, and local directories.
- * @param options The options to use for the upload.
- * @param datasets The datasets to generate the API files from.
- */
-export declare function uploadFiles(dest: string, options: UploadApiOptions, serializedFiles: AsyncIterable<SerializedFile[]>): Promise<void>;
-/**
- * Uploads the given serialized files using the given uploader.
- * @param uploader The uploader to use.
- * @param options The options to use for the upload.
- * @param datasets The datasets to generate the API files from.
- */
-export declare function uploadFilesUsingUploader(uploader: Uploader, options: UploadApiOptions, serializedFiles: AsyncIterable<SerializedFile[]>): Promise<void>;
-//# sourceMappingURL=uploads.d.ts.map

package/uploads.js

@@ -1,180 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.uploadApiFilesFromDatabase = uploadApiFilesFromDatabase;
-exports.serializeAndUploadDatasets = serializeAndUploadDatasets;
-exports.uploadFiles = uploadFiles;
-exports.uploadFilesUsingUploader = uploadFilesUsingUploader;
-const db_1 = require("./db");
-const s3_1 = require("./s3");
-const path_1 = require("path");
-const files_1 = require("./files");
-const node_stream_1 = require("node:stream");
-/**
- * Loads and generates the API files from the database and uploads them to the specified destination.
- * @param db The database that the datasets should be loaded from.
- * @param dest The destination to upload the API files to. Supported destinations are S3, zip files, and local directories.
- * @param options The options to use for the upload.
- */
-async function uploadApiFilesFromDatabase(db, dest, options) {
-    if (options.overwrite) {
-        console.log('Overwriting existing files');
-    }
-    if (options.overwriteCommonFiles) {
-        console.log('Overwriting only common files');
-    }
-    if (!!options.filePattern) {
-        console.log('Using file pattern:', options.filePattern);
-    }
-    if (options.translations) {
-        console.log('Generating for specific translations:', options.translations);
-    }
-    else {
-        console.log('Generating for all translations');
-    }
-    if (options.pretty) {
-        console.log('Generating pretty-printed JSON files');
-    }
-    const pageSize = typeof options.batchSize === 'number'
-        ? options.batchSize
-        : parseInt(options.batchSize);
-    await serializeAndUploadDatasets(dest, (0, db_1.loadDatasets)(db, pageSize, options.translations), options);
-}
-/**
- * Generates the API files from the given datasets and uploads them to the specified destination.
- * @param dest The destination to upload the API files to. Supported destinations are S3, zip files, and local directories.
- * @param options The options to use for the upload.
- * @param datasets The datasets to generate the API files from.
- */
-async function serializeAndUploadDatasets(dest, datasets, options = {}) {
-    const overwrite = !!options.overwrite;
-    if (overwrite) {
-        console.log('Overwriting existing files');
-    }
-    const overwriteCommonFiles = !!options.overwriteCommonFiles;
-    if (overwriteCommonFiles) {
-        console.log('Overwriting only common files');
-    }
-    let filePattern;
-    if (!!options.filePattern) {
-        filePattern = new RegExp(options.filePattern, 'g');
-        console.log('Using file pattern:', filePattern);
-    }
-    if (options.translations) {
-        console.log('Generating for specific translations:', options.translations);
-    }
-    else {
-        console.log('Generating for all translations');
-    }
-    if (options.pretty) {
-        console.log('Generating pretty-printed JSON files');
-    }
-    const files = (0, db_1.serializeDatasets)(datasets, {
-        ...options,
-    });
-    await uploadFiles(dest, options, files);
-}
-/**
- * Uploads the given serialized files to the specified destination.
- * @param dest The destination to upload the API files to. Supported destinations are S3, zip files, and local directories.
- * @param options The options to use for the upload.
- * @param datasets The datasets to generate the API files from.
- */
-async function uploadFiles(dest, options, serializedFiles) {
-    let uploader;
-    if (dest.startsWith('s3://')) {
-        console.log('Uploading to S3');
-        // Upload to S3
-        const url = dest;
-        const s3Url = (0, s3_1.parseS3Url)(url);
-        if (!s3Url) {
-            throw new Error(`Invalid S3 URL: ${url}`);
-        }
-        if (!s3Url.bucketName) {
-            throw new Error(`Invalid S3 URL: ${url}\nUnable to determine bucket name`);
-        }
-        uploader = new s3_1.S3Uploader(s3Url.bucketName, s3Url.objectKey, (0, s3_1.defaultProviderForOptions)(options));
-    }
-    else if (dest.startsWith('console://')) {
-        console.log('Uploading to console');
-        uploader = {
-            idealBatchSize: 50,
-            async upload(file, _overwrite) {
-                console.log(file.path);
-                console.log(file.content);
-                return true;
-            },
-        };
-    }
-    else if ((0, path_1.extname)(dest) === '.zip') {
-        console.log('Writing to zip file:', dest);
-        uploader = new files_1.ZipUploader(dest);
-    }
-    else if (dest) {
-        console.log('Writing to local directory:', dest);
-        uploader = new files_1.FilesUploader(dest);
-    }
-    else {
-        console.error('Unsupported destination:', dest);
-        process.exit(1);
-    }
-    try {
-        await uploadFilesUsingUploader(uploader, options, serializedFiles);
-    }
-    finally {
-        if (uploader && uploader.dispose) {
-            await uploader.dispose();
-        }
-    }
-}
-/**
- * Uploads the given serialized files using the given uploader.
- * @param uploader The uploader to use.
- * @param options The options to use for the upload.
- * @param datasets The datasets to generate the API files from.
- */
-async function uploadFilesUsingUploader(uploader, options, serializedFiles) {
-    const overwrite = !!options.overwrite;
-    const overwriteCommonFiles = !!options.overwriteCommonFiles;
-    let filePattern;
-    if (!!options.filePattern) {
-        filePattern = new RegExp(options.filePattern, 'g');
-    }
-    for await (let files of serializedFiles) {
-        const batchSize = uploader.idealBatchSize ?? files.length;
-        const totalBatches = Math.ceil(files.length / batchSize);
-        console.log('Uploading', files.length, 'total files');
-        console.log('Uploading in batches of', batchSize);
-        let offset = 0;
-        let batchNumber = 1;
-        let batch = files.slice(offset, offset + batchSize);
-        while (batch.length > 0) {
-            console.log('Uploading batch', batchNumber, 'of', totalBatches);
-            let writtenFiles = 0;
-            const promises = batch.map(async (file) => {
-                if (filePattern) {
-                    if (!filePattern.test(file.path)) {
-                        console.log('Skipping file:', file.path);
-                        return;
-                    }
-                }
-                const isAvailableTranslations = file.path.endsWith('available_translations.json');
-                const isCommonFile = !isAvailableTranslations;
-                if (await uploader.upload(file, overwrite || (overwriteCommonFiles && isCommonFile))) {
-                    writtenFiles++;
-                }
-                else {
-                    console.warn('File already exists:', file.path);
-                    console.warn('Skipping file');
-                }
-                if (file.content instanceof node_stream_1.Readable) {
-                    file.content.destroy();
-                }
-            });
-            await Promise.all(promises);
-            console.log('Wrote', writtenFiles, 'files');
-            batchNumber++;
-            offset += batchSize;
-            batch = files.slice(offset, offset + batchSize);
-        }
-    }
-}