@contentstack/cli-cm-export-query 1.0.0-beta.2 → 1.0.0-beta.3

package/lib/core/query-executor.js

@@ -225,19 +225,97 @@ class QueryExporter {
     async exportReferencedAssets() {
         (0, logger_1.log)(this.exportQueryConfig, 'Starting export of referenced assets...', 'info');
         try {
+            const assetsDir = path.join((0, cli_utilities_1.sanitizePath)(this.exportQueryConfig.exportDir), (0, cli_utilities_1.sanitizePath)(this.exportQueryConfig.branchName || ''), 'assets');
+            const metadataFilePath = path.join(assetsDir, 'metadata.json');
+            const assetFilePath = path.join(assetsDir, 'assets.json');
+            // Define temp file paths
+            const tempMetadataFilePath = path.join(assetsDir, 'metadata_temp.json');
+            const tempAssetFilePath = path.join(assetsDir, 'assets_temp.json');
             const assetHandler = new utils_4.AssetReferenceHandler(this.exportQueryConfig);
             // Extract referenced asset UIDs from all entries
             const assetUIDs = assetHandler.extractReferencedAssets();
             if (assetUIDs.length > 0) {
-                (0, logger_1.log)(this.exportQueryConfig, `Exporting ${assetUIDs.length} referenced assets...`, 'info');
-                const query = {
-                    modules: {
-                        assets: {
-                            uid: { $in: assetUIDs },
+                (0, logger_1.log)(this.exportQueryConfig, `Found ${assetUIDs.length} referenced assets to export`, 'info');
+                // Define batch size - can be configurable through exportQueryConfig
+                const batchSize = this.exportQueryConfig.assetBatchSize || 100;
+                if (assetUIDs.length <= batchSize) {
+                    const query = {
+                        modules: {
+                            assets: {
+                                uid: { $in: assetUIDs },
+                            },
                         },
-                    },
-                };
-                await this.moduleExporter.exportModule('assets', { query });
+                    };
+                    await this.moduleExporter.exportModule('assets', { query });
+                }
+                // if asset size is bigger than batch size, then we need to export in batches
+                // Calculate number of batches
+                const totalBatches = Math.ceil(assetUIDs.length / batchSize);
+                (0, logger_1.log)(this.exportQueryConfig, `Processing assets in ${totalBatches} batches of ${batchSize}`, 'info');
+                // Process assets in batches
+                for (let i = 0; i < totalBatches; i++) {
+                    const start = i * batchSize;
+                    const end = Math.min(start + batchSize, assetUIDs.length);
+                    const batchAssetUIDs = assetUIDs.slice(start, end);
+                    (0, logger_1.log)(this.exportQueryConfig, `Exporting batch ${i + 1}/${totalBatches} (${batchAssetUIDs.length} assets)...`, 'info');
+                    const query = {
+                        modules: {
+                            assets: {
+                                uid: { $in: batchAssetUIDs },
+                            },
+                        },
+                    };
+                    await this.moduleExporter.exportModule('assets', { query });
+                    // Read the current batch's metadata.json and assets.json files
+                    const currentMetadata = utils_2.fsUtil.readFile((0, cli_utilities_1.sanitizePath)(metadataFilePath));
+                    const currentAssets = utils_2.fsUtil.readFile((0, cli_utilities_1.sanitizePath)(assetFilePath));
+                    // Check if this is the first batch
+                    if (i === 0) {
+                        // For first batch, initialize temp files with current content
+                        utils_2.fsUtil.writeFile((0, cli_utilities_1.sanitizePath)(tempMetadataFilePath), currentMetadata);
+                        utils_2.fsUtil.writeFile((0, cli_utilities_1.sanitizePath)(tempAssetFilePath), currentAssets);
+                        (0, logger_1.log)(this.exportQueryConfig, `Initialized temporary files with first batch data`, 'info');
+                    }
+                    else {
+                        // For subsequent batches, append to temp files with incremented keys
+                        // Handle metadata (which contains arrays of asset info)
+                        const tempMetadata = utils_2.fsUtil.readFile((0, cli_utilities_1.sanitizePath)(tempMetadataFilePath)) || {};
+                        // Merge metadata by combining arrays
+                        if (currentMetadata) {
+                            Object.keys(currentMetadata).forEach((key) => {
+                                if (!tempMetadata[key]) {
+                                    tempMetadata[key] = currentMetadata[key];
+                                }
+                            });
+                        }
+                        // Write updated metadata back to temp file
+                        utils_2.fsUtil.writeFile((0, cli_utilities_1.sanitizePath)(tempMetadataFilePath), tempMetadata);
+                        // Handle assets (which is an object with numeric keys)
+                        const tempAssets = utils_2.fsUtil.readFile((0, cli_utilities_1.sanitizePath)(tempAssetFilePath)) || {};
+                        let nextIndex = Object.keys(tempAssets).length + 1;
+                        // Add current assets with incremented keys
+                        Object.values(currentAssets).forEach((value) => {
+                            tempAssets[nextIndex.toString()] = value;
+                            nextIndex++;
+                        });
+                        utils_2.fsUtil.writeFile((0, cli_utilities_1.sanitizePath)(tempAssetFilePath), tempAssets);
+                        (0, logger_1.log)(this.exportQueryConfig, `Updated temporary files with batch ${i + 1} data`, 'info');
+                    }
+                    // Optional: Add delay between batches to avoid rate limiting
+                    if (i < totalBatches - 1 && this.exportQueryConfig.batchDelayMs) {
+                        await new Promise((resolve) => setTimeout(resolve, this.exportQueryConfig.batchDelayMs));
+                    }
+                }
+                // After all batches are processed, copy temp files back to original files
+                const finalMetadata = utils_2.fsUtil.readFile((0, cli_utilities_1.sanitizePath)(tempMetadataFilePath));
+                const finalAssets = utils_2.fsUtil.readFile((0, cli_utilities_1.sanitizePath)(tempAssetFilePath));
+                utils_2.fsUtil.writeFile((0, cli_utilities_1.sanitizePath)(metadataFilePath), finalMetadata);
+                utils_2.fsUtil.writeFile((0, cli_utilities_1.sanitizePath)(assetFilePath), finalAssets);
+                (0, logger_1.log)(this.exportQueryConfig, `Final data written back to original files`, 'info');
+                // Clean up temp files
+                utils_2.fsUtil.removeFile((0, cli_utilities_1.sanitizePath)(tempMetadataFilePath));
+                utils_2.fsUtil.removeFile((0, cli_utilities_1.sanitizePath)(tempAssetFilePath));
+                (0, logger_1.log)(this.exportQueryConfig, `Temporary files cleaned up`, 'info');
                 (0, logger_1.log)(this.exportQueryConfig, 'Referenced assets exported successfully', 'success');
             }
             else {

package/lib/types/index.d.ts

@@ -140,6 +140,9 @@ export interface QueryExportConfig extends DefaultConfig {
     logsPath: string;
     dataPath: string;
     exportDelayMs?: number;
+    batchDelayMs?: number;
+    assetBatchSize?: number;
+    assetBatchDelayMs?: number;
 }
 export interface QueryMetadata {
     query: any;

package/lib/utils/config-handler.js

@@ -12,6 +12,10 @@ async function setupQueryExportConfig(flags) {
     const exportQueryConfig = Object.assign(Object.assign({}, config_1.default), { exportDir, stackApiKey: flags['stack-api-key'] || '', managementToken: flags.alias ? (_a = cli_utilities_1.configHandler.get(`tokens.${flags.alias}`)) === null || _a === void 0 ? void 0 : _a.token : undefined, query: flags.query, skipReferences: flags['skip-references'] || false, skipDependencies: flags['skip-dependencies'] || false, branchName: flags.branch, securedAssets: flags['secured-assets'] || false, isQueryBasedExport: true, logsPath: exportDir, dataPath: exportDir, 
         // Todo: accept the path of the config file from the user
         externalConfigPath: path.join(__dirname, '../config/export-config.json') });
+    // override the external config path if the user provides a config file
+    if (flags.config) {
+        exportQueryConfig.externalConfigPath = (0, cli_utilities_1.sanitizePath)(flags['config']);
+    }
     // Handle authentication
     if (flags.alias) {
         const { token, apiKey } = cli_utilities_1.configHandler.get(`tokens.${flags.alias}`) || {};

package/oclif.manifest.json

@@ -102,5 +102,5 @@
       ]
     }
   },
-  "version": "1.0.0-beta.2"
+  "version": "1.0.0-beta.3"
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@contentstack/cli-cm-export-query",
   "description": "Contentstack CLI plugin to export content from stack",
-  "version": "1.0.0-beta.2",
+  "version": "1.0.0-beta.3",
   "author": "Contentstack",
   "bugs": "https://github.com/contentstack/cli/issues",
   "dependencies": {