@contentstack/cli-cm-export-query 1.0.0-beta.2 → 1.0.0-beta.4

package/lib/commands/cm/stacks/export-query.js

@@ -65,6 +65,11 @@ ExportQueryCommand.flags = {
     }),
     branch: cli_utilities_1.flags.string({
         description: 'Branch name to export from',
+        exclusive: ['branch-alias'],
+    }),
+    'branch-alias': cli_utilities_1.flags.string({
+        description: 'Alias of Branch to export from',
+        exclusive: ['branch'],
     }),
     query: cli_utilities_1.flags.string({
         required: true,

package/lib/core/query-executor.js

@@ -225,19 +225,97 @@ class QueryExporter {
     async exportReferencedAssets() {
         (0, logger_1.log)(this.exportQueryConfig, 'Starting export of referenced assets...', 'info');
         try {
+            const assetsDir = path.join((0, cli_utilities_1.sanitizePath)(this.exportQueryConfig.exportDir), (0, cli_utilities_1.sanitizePath)(this.exportQueryConfig.branchName || ''), 'assets');
+            const metadataFilePath = path.join(assetsDir, 'metadata.json');
+            const assetFilePath = path.join(assetsDir, 'assets.json');
+            // Define temp file paths
+            const tempMetadataFilePath = path.join(assetsDir, 'metadata_temp.json');
+            const tempAssetFilePath = path.join(assetsDir, 'assets_temp.json');
             const assetHandler = new utils_4.AssetReferenceHandler(this.exportQueryConfig);
             // Extract referenced asset UIDs from all entries
             const assetUIDs = assetHandler.extractReferencedAssets();
             if (assetUIDs.length > 0) {
-                (0, logger_1.log)(this.exportQueryConfig, `Exporting ${assetUIDs.length} referenced assets...`, 'info');
-                const query = {
-                    modules: {
-                        assets: {
-                            uid: { $in: assetUIDs },
+                (0, logger_1.log)(this.exportQueryConfig, `Found ${assetUIDs.length} referenced assets to export`, 'info');
+                // Define batch size - can be configurable through exportQueryConfig
+                const batchSize = this.exportQueryConfig.assetBatchSize || 100;
+                if (assetUIDs.length <= batchSize) {
+                    const query = {
+                        modules: {
+                            assets: {
+                                uid: { $in: assetUIDs },
+                            },
                         },
-                    },
-                };
-                await this.moduleExporter.exportModule('assets', { query });
+                    };
+                    await this.moduleExporter.exportModule('assets', { query });
+                }
+                // if asset size is bigger than batch size, then we need to export in batches
+                // Calculate number of batches
+                const totalBatches = Math.ceil(assetUIDs.length / batchSize);
+                (0, logger_1.log)(this.exportQueryConfig, `Processing assets in ${totalBatches} batches of ${batchSize}`, 'info');
+                // Process assets in batches
+                for (let i = 0; i < totalBatches; i++) {
+                    const start = i * batchSize;
+                    const end = Math.min(start + batchSize, assetUIDs.length);
+                    const batchAssetUIDs = assetUIDs.slice(start, end);
+                    (0, logger_1.log)(this.exportQueryConfig, `Exporting batch ${i + 1}/${totalBatches} (${batchAssetUIDs.length} assets)...`, 'info');
+                    const query = {
+                        modules: {
+                            assets: {
+                                uid: { $in: batchAssetUIDs },
+                            },
+                        },
+                    };
+                    await this.moduleExporter.exportModule('assets', { query });
+                    // Read the current batch's metadata.json and assets.json files
+                    const currentMetadata = utils_2.fsUtil.readFile((0, cli_utilities_1.sanitizePath)(metadataFilePath));
+                    const currentAssets = utils_2.fsUtil.readFile((0, cli_utilities_1.sanitizePath)(assetFilePath));
+                    // Check if this is the first batch
+                    if (i === 0) {
+                        // For first batch, initialize temp files with current content
+                        utils_2.fsUtil.writeFile((0, cli_utilities_1.sanitizePath)(tempMetadataFilePath), currentMetadata);
+                        utils_2.fsUtil.writeFile((0, cli_utilities_1.sanitizePath)(tempAssetFilePath), currentAssets);
+                        (0, logger_1.log)(this.exportQueryConfig, `Initialized temporary files with first batch data`, 'info');
+                    }
+                    else {
+                        // For subsequent batches, append to temp files with incremented keys
+                        // Handle metadata (which contains arrays of asset info)
+                        const tempMetadata = utils_2.fsUtil.readFile((0, cli_utilities_1.sanitizePath)(tempMetadataFilePath)) || {};
+                        // Merge metadata by combining arrays
+                        if (currentMetadata) {
+                            Object.keys(currentMetadata).forEach((key) => {
+                                if (!tempMetadata[key]) {
+                                    tempMetadata[key] = currentMetadata[key];
+                                }
+                            });
+                        }
+                        // Write updated metadata back to temp file
+                        utils_2.fsUtil.writeFile((0, cli_utilities_1.sanitizePath)(tempMetadataFilePath), tempMetadata);
+                        // Handle assets (which is an object with numeric keys)
+                        const tempAssets = utils_2.fsUtil.readFile((0, cli_utilities_1.sanitizePath)(tempAssetFilePath)) || {};
+                        let nextIndex = Object.keys(tempAssets).length + 1;
+                        // Add current assets with incremented keys
+                        Object.values(currentAssets).forEach((value) => {
+                            tempAssets[nextIndex.toString()] = value;
+                            nextIndex++;
+                        });
+                        utils_2.fsUtil.writeFile((0, cli_utilities_1.sanitizePath)(tempAssetFilePath), tempAssets);
+                        (0, logger_1.log)(this.exportQueryConfig, `Updated temporary files with batch ${i + 1} data`, 'info');
+                    }
+                    // Optional: Add delay between batches to avoid rate limiting
+                    if (i < totalBatches - 1 && this.exportQueryConfig.batchDelayMs) {
+                        await new Promise((resolve) => setTimeout(resolve, this.exportQueryConfig.batchDelayMs));
+                    }
+                }
+                // After all batches are processed, copy temp files back to original files
+                const finalMetadata = utils_2.fsUtil.readFile((0, cli_utilities_1.sanitizePath)(tempMetadataFilePath));
+                const finalAssets = utils_2.fsUtil.readFile((0, cli_utilities_1.sanitizePath)(tempAssetFilePath));
+                utils_2.fsUtil.writeFile((0, cli_utilities_1.sanitizePath)(metadataFilePath), finalMetadata);
+                utils_2.fsUtil.writeFile((0, cli_utilities_1.sanitizePath)(assetFilePath), finalAssets);
+                (0, logger_1.log)(this.exportQueryConfig, `Final data written back to original files`, 'info');
+                // Clean up temp files
+                utils_2.fsUtil.removeFile((0, cli_utilities_1.sanitizePath)(tempMetadataFilePath));
+                utils_2.fsUtil.removeFile((0, cli_utilities_1.sanitizePath)(tempAssetFilePath));
+                (0, logger_1.log)(this.exportQueryConfig, `Temporary files cleaned up`, 'info');
                 (0, logger_1.log)(this.exportQueryConfig, 'Referenced assets exported successfully', 'success');
             }
             else {

package/lib/types/index.d.ts

@@ -136,10 +136,14 @@ export interface QueryExportConfig extends DefaultConfig {
     stackApiKey: string;
     managementToken?: string;
     branchName: string;
+    branchAlias?: string;
     securedAssets: boolean;
     logsPath: string;
     dataPath: string;
     exportDelayMs?: number;
+    batchDelayMs?: number;
+    assetBatchSize?: number;
+    assetBatchDelayMs?: number;
 }
 export interface QueryMetadata {
     query: any;

package/lib/utils/branch-helper.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.setupBranches = void 0;
+const cli_utilities_1 = require("@contentstack/cli-utilities");
 const logger_1 = require("./logger");
 /**
  * Validates and sets up branch configuration for the stack
@@ -14,6 +15,10 @@ const setupBranches = async (config, stackAPIClient) => {
         throw new Error('Invalid config to setup the branch');
     }
     try {
+        if (config.branchAlias) {
+            config.branchName = await (0, cli_utilities_1.getBranchFromAlias)(stackAPIClient, config.branchAlias);
+            return;
+        }
         if (config.branchName) {
             // Check if the specified branch exists
             (0, logger_1.log)(config, `Validating branch: ${config.branchName}`, 'info');

package/lib/utils/config-handler.js

@@ -12,6 +12,13 @@ async function setupQueryExportConfig(flags) {
     const exportQueryConfig = Object.assign(Object.assign({}, config_1.default), { exportDir, stackApiKey: flags['stack-api-key'] || '', managementToken: flags.alias ? (_a = cli_utilities_1.configHandler.get(`tokens.${flags.alias}`)) === null || _a === void 0 ? void 0 : _a.token : undefined, query: flags.query, skipReferences: flags['skip-references'] || false, skipDependencies: flags['skip-dependencies'] || false, branchName: flags.branch, securedAssets: flags['secured-assets'] || false, isQueryBasedExport: true, logsPath: exportDir, dataPath: exportDir, 
         // Todo: accept the path of the config file from the user
         externalConfigPath: path.join(__dirname, '../config/export-config.json') });
+    if (flags['branch-alias']) {
+        exportQueryConfig.branchAlias = flags['branch-alias'];
+    }
+    // override the external config path if the user provides a config file
+    if (flags.config) {
+        exportQueryConfig.externalConfigPath = (0, cli_utilities_1.sanitizePath)(flags['config']);
+    }
     // Handle authentication
     if (flags.alias) {
         const { token, apiKey } = cli_utilities_1.configHandler.get(`tokens.${flags.alias}`) || {};

package/oclif.manifest.json

@@ -45,11 +45,24 @@
         },
         "branch": {
           "description": "Branch name to export from",
+          "exclusive": [
+            "branch-alias"
+          ],
           "name": "branch",
           "hasDynamicHelp": false,
           "multiple": false,
           "type": "option"
         },
+        "branch-alias": {
+          "description": "Alias of Branch to export from",
+          "exclusive": [
+            "branch"
+          ],
+          "name": "branch-alias",
+          "hasDynamicHelp": false,
+          "multiple": false,
+          "type": "option"
+        },
         "query": {
           "description": "Query as JSON string or file path",
           "name": "query",
@@ -102,5 +115,5 @@
       ]
     }
   },
-  "version": "1.0.0-beta.2"
+  "version": "1.0.0-beta.4"
 }

package/package.json

@@ -1,13 +1,13 @@
 {
   "name": "@contentstack/cli-cm-export-query",
   "description": "Contentstack CLI plugin to export content from stack",
-  "version": "1.0.0-beta.2",
+  "version": "1.0.0-beta.4",
   "author": "Contentstack",
   "bugs": "https://github.com/contentstack/cli/issues",
   "dependencies": {
-    "@contentstack/cli-cm-export": "~1.18.0",
-    "@contentstack/cli-command": "~1.6.0",
-    "@contentstack/cli-utilities": "~1.13.0",
+    "@contentstack/cli-cm-export": "~1.20.2",
+    "@contentstack/cli-command": "~1.6.1",
+    "@contentstack/cli-utilities": "~1.14.1",
     "@oclif/core": "^4.3.0",
     "async": "^3.2.6",
     "big-json": "^3.2.0",
@@ -21,21 +21,21 @@
     "winston": "^3.17.0"
   },
   "devDependencies": {
-    "@contentstack/cli-dev-dependencies": "~1.3.0",
-    "@oclif/plugin-help": "^6.2.28",
-    "@oclif/test": "^4.1.13",
+    "@contentstack/cli-dev-dependencies": "~1.3.1",
+    "@oclif/plugin-help": "^6.2.33",
+    "@oclif/test": "^4.1.14",
     "@types/big-json": "^3.2.5",
     "@types/chai": "^4.3.20",
     "@types/mkdirp": "^1.0.2",
     "@types/mocha": "^10.0.10",
-    "@types/node": "^20.19.8",
+    "@types/node": "^20.19.17",
     "@types/progress-stream": "^2.0.5",
     "@types/sinon": "^17.0.4",
     "chai": "^4.5.0",
-    "dotenv": "^16.5.0",
+    "dotenv": "^16.6.1",
     "dotenv-expand": "^9.0.0",
     "eslint": "^8.57.1",
-    "eslint-config-oclif": "^6.0.62",
+    "eslint-config-oclif": "^6.0.104",
     "husky": "^9.1.7",
     "mocha": "10.8.2",
     "nyc": "^15.1.0",