@helloao/cli 0.0.6 → 0.0.8-alpha

package/uploads.js

@@ -1,180 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.uploadApiFilesFromDatabase = uploadApiFilesFromDatabase;
-exports.serializeAndUploadDatasets = serializeAndUploadDatasets;
-exports.uploadFiles = uploadFiles;
-exports.uploadFilesUsingUploader = uploadFilesUsingUploader;
-const db_1 = require("./db");
-const s3_1 = require("./s3");
-const path_1 = require("path");
-const files_1 = require("./files");
-const node_stream_1 = require("node:stream");
-/**
- * Loads and generates the API files from the database and uploads them to the specified destination.
- * @param db The database that the datasets should be loaded from.
- * @param dest The destination to upload the API files to. Supported destinations are S3, zip files, and local directories.
- * @param options The options to use for the upload.
- */
-async function uploadApiFilesFromDatabase(db, dest, options) {
-    if (options.overwrite) {
-        console.log('Overwriting existing files');
-    }
-    if (options.overwriteCommonFiles) {
-        console.log('Overwriting only common files');
-    }
-    if (!!options.filePattern) {
-        console.log('Using file pattern:', options.filePattern);
-    }
-    if (options.translations) {
-        console.log('Generating for specific translations:', options.translations);
-    }
-    else {
-        console.log('Generating for all translations');
-    }
-    if (options.pretty) {
-        console.log('Generating pretty-printed JSON files');
-    }
-    const pageSize = typeof options.batchSize === 'number'
-        ? options.batchSize
-        : parseInt(options.batchSize);
-    await serializeAndUploadDatasets(dest, (0, db_1.loadDatasets)(db, pageSize, options.translations), options);
-}
-/**
- * Generates the API files from the given datasets and uploads them to the specified destination.
- * @param dest The destination to upload the API files to. Supported destinations are S3, zip files, and local directories.
- * @param options The options to use for the upload.
- * @param datasets The datasets to generate the API files from.
- */
-async function serializeAndUploadDatasets(dest, datasets, options = {}) {
-    const overwrite = !!options.overwrite;
-    if (overwrite) {
-        console.log('Overwriting existing files');
-    }
-    const overwriteCommonFiles = !!options.overwriteCommonFiles;
-    if (overwriteCommonFiles) {
-        console.log('Overwriting only common files');
-    }
-    let filePattern;
-    if (!!options.filePattern) {
-        filePattern = new RegExp(options.filePattern, 'g');
-        console.log('Using file pattern:', filePattern);
-    }
-    if (options.translations) {
-        console.log('Generating for specific translations:', options.translations);
-    }
-    else {
-        console.log('Generating for all translations');
-    }
-    if (options.pretty) {
-        console.log('Generating pretty-printed JSON files');
-    }
-    const files = (0, db_1.serializeDatasets)(datasets, {
-        ...options,
-    });
-    await uploadFiles(dest, options, files);
-}
-/**
- * Uploads the given serialized files to the specified destination.
- * @param dest The destination to upload the API files to. Supported destinations are S3, zip files, and local directories.
- * @param options The options to use for the upload.
- * @param datasets The datasets to generate the API files from.
- */
-async function uploadFiles(dest, options, serializedFiles) {
-    let uploader;
-    if (dest.startsWith('s3://')) {
-        console.log('Uploading to S3');
-        // Upload to S3
-        const url = dest;
-        const s3Url = (0, s3_1.parseS3Url)(url);
-        if (!s3Url) {
-            throw new Error(`Invalid S3 URL: ${url}`);
-        }
-        if (!s3Url.bucketName) {
-            throw new Error(`Invalid S3 URL: ${url}\nUnable to determine bucket name`);
-        }
-        uploader = new s3_1.S3Uploader(s3Url.bucketName, s3Url.objectKey, (0, s3_1.defaultProviderForOptions)(options));
-    }
-    else if (dest.startsWith('console://')) {
-        console.log('Uploading to console');
-        uploader = {
-            idealBatchSize: 50,
-            async upload(file, _overwrite) {
-                console.log(file.path);
-                console.log(file.content);
-                return true;
-            },
-        };
-    }
-    else if ((0, path_1.extname)(dest) === '.zip') {
-        console.log('Writing to zip file:', dest);
-        uploader = new files_1.ZipUploader(dest);
-    }
-    else if (dest) {
-        console.log('Writing to local directory:', dest);
-        uploader = new files_1.FilesUploader(dest);
-    }
-    else {
-        console.error('Unsupported destination:', dest);
-        process.exit(1);
-    }
-    try {
-        await uploadFilesUsingUploader(uploader, options, serializedFiles);
-    }
-    finally {
-        if (uploader && uploader.dispose) {
-            await uploader.dispose();
-        }
-    }
-}
-/**
- * Uploads the given serialized files using the given uploader.
- * @param uploader The uploader to use.
- * @param options The options to use for the upload.
- * @param datasets The datasets to generate the API files from.
- */
-async function uploadFilesUsingUploader(uploader, options, serializedFiles) {
-    const overwrite = !!options.overwrite;
-    const overwriteCommonFiles = !!options.overwriteCommonFiles;
-    let filePattern;
-    if (!!options.filePattern) {
-        filePattern = new RegExp(options.filePattern, 'g');
-    }
-    for await (let files of serializedFiles) {
-        const batchSize = uploader.idealBatchSize ?? files.length;
-        const totalBatches = Math.ceil(files.length / batchSize);
-        console.log('Uploading', files.length, 'total files');
-        console.log('Uploading in batches of', batchSize);
-        let offset = 0;
-        let batchNumber = 1;
-        let batch = files.slice(offset, offset + batchSize);
-        while (batch.length > 0) {
-            console.log('Uploading batch', batchNumber, 'of', totalBatches);
-            let writtenFiles = 0;
-            const promises = batch.map(async (file) => {
-                if (filePattern) {
-                    if (!filePattern.test(file.path)) {
-                        console.log('Skipping file:', file.path);
-                        return;
-                    }
-                }
-                const isAvailableTranslations = file.path.endsWith('available_translations.json');
-                const isCommonFile = !isAvailableTranslations;
-                if (await uploader.upload(file, overwrite || (overwriteCommonFiles && isCommonFile))) {
-                    writtenFiles++;
-                }
-                else {
-                    console.warn('File already exists:', file.path);
-                    console.warn('Skipping file');
-                }
-                if (file.content instanceof node_stream_1.Readable) {
-                    file.content.destroy();
-                }
-            });
-            await Promise.all(promises);
-            console.log('Wrote', writtenFiles, 'files');
-            batchNumber++;
-            offset += batchSize;
-            batch = files.slice(offset, offset + batchSize);
-        }
-    }
-}