@contentstack/cli-cm-export-query 1.0.0-beta.1 → 1.0.0-beta.3

package/lib/commands/cm/stacks/export-query.d.ts

@@ -6,6 +6,5 @@ export default class ExportQueryCommand extends Command {
     static examples: string[];
     static usage: string;
     static flags: FlagInput;
-    static aliases: string[];
     run(): Promise<void>;
 }

package/lib/commands/cm/stacks/export-query.js

@@ -16,8 +16,16 @@ class ExportQueryCommand extends cli_command_1.Command {
                 exportQueryConfig.developerHubBaseUrl = this.developerHubUrl;
             }
             this.exportDir = (0, cli_utilities_1.sanitizePath)(exportQueryConfig.exportDir);
-            // Initialize and run query export
+            // Initialize management API client
             const managementAPIClient = await (0, cli_utilities_1.managementSDKClient)(exportQueryConfig);
+            // Setup and validate branch configuration
+            const stackAPIClient = managementAPIClient.stack({
+                api_key: exportQueryConfig.stackApiKey,
+                management_token: exportQueryConfig.managementToken,
+            });
+            // Setup branches (validate branch or set default to 'main')
+            await (0, utils_1.setupBranches)(exportQueryConfig, stackAPIClient);
+            // Initialize and run query export
             const queryExporter = new query_executor_1.QueryExporter(managementAPIClient, exportQueryConfig);
             await queryExporter.execute();
             (0, utils_1.log)(exportQueryConfig, 'Query-based export completed successfully!', 'success');
@@ -76,4 +84,3 @@ ExportQueryCommand.flags = {
         description: 'Skip confirmation prompts',
     }),
 };
-ExportQueryCommand.aliases = ['cm:export-query'];

package/lib/core/module-exporter.js

@@ -15,7 +15,6 @@ class ModuleExporter {
             (0, logger_1.log)(this.exportQueryConfig, `Exporting module: ${moduleName}`, 'info');
             // Build command arguments
             const cmd = this.buildExportCommand(moduleName, options);
-            (0, logger_1.log)(this.exportQueryConfig, `Running export command: ${cmd.join(' ')}`, 'debug');
             // Configurable delay
             const delay = this.exportQueryConfig.exportDelayMs || 2000;
             await new Promise((resolve) => setTimeout(resolve, delay));

package/lib/core/query-executor.js

@@ -225,19 +225,97 @@ class QueryExporter {
     async exportReferencedAssets() {
         (0, logger_1.log)(this.exportQueryConfig, 'Starting export of referenced assets...', 'info');
         try {
+            const assetsDir = path.join((0, cli_utilities_1.sanitizePath)(this.exportQueryConfig.exportDir), (0, cli_utilities_1.sanitizePath)(this.exportQueryConfig.branchName || ''), 'assets');
+            const metadataFilePath = path.join(assetsDir, 'metadata.json');
+            const assetFilePath = path.join(assetsDir, 'assets.json');
+            // Define temp file paths
+            const tempMetadataFilePath = path.join(assetsDir, 'metadata_temp.json');
+            const tempAssetFilePath = path.join(assetsDir, 'assets_temp.json');
             const assetHandler = new utils_4.AssetReferenceHandler(this.exportQueryConfig);
             // Extract referenced asset UIDs from all entries
             const assetUIDs = assetHandler.extractReferencedAssets();
             if (assetUIDs.length > 0) {
-                (0, logger_1.log)(this.exportQueryConfig, `Exporting ${assetUIDs.length} referenced assets...`, 'info');
-                const query = {
-                    modules: {
-                        assets: {
-                            uid: { $in: assetUIDs },
+                (0, logger_1.log)(this.exportQueryConfig, `Found ${assetUIDs.length} referenced assets to export`, 'info');
+                // Define batch size - can be configurable through exportQueryConfig
+                const batchSize = this.exportQueryConfig.assetBatchSize || 100;
+                if (assetUIDs.length <= batchSize) {
+                    const query = {
+                        modules: {
+                            assets: {
+                                uid: { $in: assetUIDs },
+                            },
                         },
-                    },
-                };
-                await this.moduleExporter.exportModule('assets', { query });
+                    };
+                    await this.moduleExporter.exportModule('assets', { query });
+                }
+                // if asset size is bigger than batch size, then we need to export in batches
+                // Calculate number of batches
+                const totalBatches = Math.ceil(assetUIDs.length / batchSize);
+                (0, logger_1.log)(this.exportQueryConfig, `Processing assets in ${totalBatches} batches of ${batchSize}`, 'info');
+                // Process assets in batches
+                for (let i = 0; i < totalBatches; i++) {
+                    const start = i * batchSize;
+                    const end = Math.min(start + batchSize, assetUIDs.length);
+                    const batchAssetUIDs = assetUIDs.slice(start, end);
+                    (0, logger_1.log)(this.exportQueryConfig, `Exporting batch ${i + 1}/${totalBatches} (${batchAssetUIDs.length} assets)...`, 'info');
+                    const query = {
+                        modules: {
+                            assets: {
+                                uid: { $in: batchAssetUIDs },
+                            },
+                        },
+                    };
+                    await this.moduleExporter.exportModule('assets', { query });
+                    // Read the current batch's metadata.json and assets.json files
+                    const currentMetadata = utils_2.fsUtil.readFile((0, cli_utilities_1.sanitizePath)(metadataFilePath));
+                    const currentAssets = utils_2.fsUtil.readFile((0, cli_utilities_1.sanitizePath)(assetFilePath));
+                    // Check if this is the first batch
+                    if (i === 0) {
+                        // For first batch, initialize temp files with current content
+                        utils_2.fsUtil.writeFile((0, cli_utilities_1.sanitizePath)(tempMetadataFilePath), currentMetadata);
+                        utils_2.fsUtil.writeFile((0, cli_utilities_1.sanitizePath)(tempAssetFilePath), currentAssets);
+                        (0, logger_1.log)(this.exportQueryConfig, `Initialized temporary files with first batch data`, 'info');
+                    }
+                    else {
+                        // For subsequent batches, append to temp files with incremented keys
+                        // Handle metadata (which contains arrays of asset info)
+                        const tempMetadata = utils_2.fsUtil.readFile((0, cli_utilities_1.sanitizePath)(tempMetadataFilePath)) || {};
+                        // Merge metadata by combining arrays
+                        if (currentMetadata) {
+                            Object.keys(currentMetadata).forEach((key) => {
+                                if (!tempMetadata[key]) {
+                                    tempMetadata[key] = currentMetadata[key];
+                                }
+                            });
+                        }
+                        // Write updated metadata back to temp file
+                        utils_2.fsUtil.writeFile((0, cli_utilities_1.sanitizePath)(tempMetadataFilePath), tempMetadata);
+                        // Handle assets (which is an object with numeric keys)
+                        const tempAssets = utils_2.fsUtil.readFile((0, cli_utilities_1.sanitizePath)(tempAssetFilePath)) || {};
+                        let nextIndex = Object.keys(tempAssets).length + 1;
+                        // Add current assets with incremented keys
+                        Object.values(currentAssets).forEach((value) => {
+                            tempAssets[nextIndex.toString()] = value;
+                            nextIndex++;
+                        });
+                        utils_2.fsUtil.writeFile((0, cli_utilities_1.sanitizePath)(tempAssetFilePath), tempAssets);
+                        (0, logger_1.log)(this.exportQueryConfig, `Updated temporary files with batch ${i + 1} data`, 'info');
+                    }
+                    // Optional: Add delay between batches to avoid rate limiting
+                    if (i < totalBatches - 1 && this.exportQueryConfig.batchDelayMs) {
+                        await new Promise((resolve) => setTimeout(resolve, this.exportQueryConfig.batchDelayMs));
+                    }
+                }
+                // After all batches are processed, copy temp files back to original files
+                const finalMetadata = utils_2.fsUtil.readFile((0, cli_utilities_1.sanitizePath)(tempMetadataFilePath));
+                const finalAssets = utils_2.fsUtil.readFile((0, cli_utilities_1.sanitizePath)(tempAssetFilePath));
+                utils_2.fsUtil.writeFile((0, cli_utilities_1.sanitizePath)(metadataFilePath), finalMetadata);
+                utils_2.fsUtil.writeFile((0, cli_utilities_1.sanitizePath)(assetFilePath), finalAssets);
+                (0, logger_1.log)(this.exportQueryConfig, `Final data written back to original files`, 'info');
+                // Clean up temp files
+                utils_2.fsUtil.removeFile((0, cli_utilities_1.sanitizePath)(tempMetadataFilePath));
+                utils_2.fsUtil.removeFile((0, cli_utilities_1.sanitizePath)(tempAssetFilePath));
+                (0, logger_1.log)(this.exportQueryConfig, `Temporary files cleaned up`, 'info');
                 (0, logger_1.log)(this.exportQueryConfig, 'Referenced assets exported successfully', 'success');
             }
             else {

package/lib/types/index.d.ts

@@ -140,6 +140,9 @@ export interface QueryExportConfig extends DefaultConfig {
     logsPath: string;
     dataPath: string;
     exportDelayMs?: number;
+    batchDelayMs?: number;
+    assetBatchSize?: number;
+    assetBatchDelayMs?: number;
 }
 export interface QueryMetadata {
     query: any;

package/lib/utils/branch-helper.d.ts

@@ -0,0 +1,10 @@
+import { QueryExportConfig } from '../types';
+/**
+ * Validates and sets up branch configuration for the stack
+ *
+ * @param config The export configuration
+ * @param stackAPIClient The stack API client
+ * @returns Promise that resolves when branch setup is complete
+ */
+export declare const setupBranches: (config: QueryExportConfig, stackAPIClient: any) => Promise<void>;
+export default setupBranches;

package/lib/utils/branch-helper.js

@@ -0,0 +1,63 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.setupBranches = void 0;
+const logger_1 = require("./logger");
+/**
+ * Validates and sets up branch configuration for the stack
+ *
+ * @param config The export configuration
+ * @param stackAPIClient The stack API client
+ * @returns Promise that resolves when branch setup is complete
+ */
+const setupBranches = async (config, stackAPIClient) => {
+    if (typeof config !== 'object') {
+        throw new Error('Invalid config to setup the branch');
+    }
+    try {
+        if (config.branchName) {
+            // Check if the specified branch exists
+            (0, logger_1.log)(config, `Validating branch: ${config.branchName}`, 'info');
+            const result = await stackAPIClient
+                .branch(config.branchName)
+                .fetch()
+                .catch((err) => {
+                (0, logger_1.log)(config, `Error fetching branch: ${err.message}`, 'error');
+                return null;
+            });
+            if (result && typeof result === 'object') {
+                (0, logger_1.log)(config, `Branch '${config.branchName}' found`, 'success');
+            }
+            else {
+                throw new Error(`No branch found with the name '${config.branchName}'`);
+            }
+        }
+        else {
+            // If no branch name provided, check if the stack has branches
+            (0, logger_1.log)(config, 'No branch specified, checking if stack has branches', 'info');
+            const result = await stackAPIClient
+                .branch()
+                .query()
+                .find()
+                .catch(() => {
+                (0, logger_1.log)(config, 'Stack does not have branches', 'info');
+                return null;
+            });
+            if (result && result.items && Array.isArray(result.items) && result.items.length > 0) {
+                // Set default branch to 'main' if it exists
+                config.branchName = 'main';
+            }
+            else {
+                // Stack doesn't have branches
+                (0, logger_1.log)(config, 'Stack does not have branches', 'info');
+                return;
+            }
+        }
+        config.branchEnabled = true;
+    }
+    catch (error) {
+        (0, logger_1.log)(config, `Error setting up branches: ${error.message}`, 'error');
+        throw error;
+    }
+};
+exports.setupBranches = setupBranches;
+exports.default = exports.setupBranches;

package/lib/utils/config-handler.js

@@ -12,22 +12,26 @@ async function setupQueryExportConfig(flags) {
     const exportQueryConfig = Object.assign(Object.assign({}, config_1.default), { exportDir, stackApiKey: flags['stack-api-key'] || '', managementToken: flags.alias ? (_a = cli_utilities_1.configHandler.get(`tokens.${flags.alias}`)) === null || _a === void 0 ? void 0 : _a.token : undefined, query: flags.query, skipReferences: flags['skip-references'] || false, skipDependencies: flags['skip-dependencies'] || false, branchName: flags.branch, securedAssets: flags['secured-assets'] || false, isQueryBasedExport: true, logsPath: exportDir, dataPath: exportDir, 
         // Todo: accept the path of the config file from the user
         externalConfigPath: path.join(__dirname, '../config/export-config.json') });
+    // override the external config path if the user provides a config file
+    if (flags.config) {
+        exportQueryConfig.externalConfigPath = (0, cli_utilities_1.sanitizePath)(flags['config']);
+    }
     // Handle authentication
     if (flags.alias) {
         const { token, apiKey } = cli_utilities_1.configHandler.get(`tokens.${flags.alias}`) || {};
-        config_1.default.managementToken = token;
-        config_1.default.stackApiKey = apiKey;
-        if (!config_1.default.managementToken) {
+        exportQueryConfig.managementToken = token;
+        exportQueryConfig.stackApiKey = apiKey;
+        if (!exportQueryConfig.managementToken) {
             throw new Error(`No management token found on given alias ${flags.alias}`);
         }
     }
-    if (!config_1.default.managementToken) {
+    if (!exportQueryConfig.managementToken) {
         if (!(0, cli_utilities_1.isAuthenticated)()) {
             throw new Error('Please login or provide an alias for the management token');
         }
         else {
-            config_1.default.stackApiKey = flags['stack-api-key'] || (await (0, common_helper_1.askAPIKey)());
-            if (typeof config_1.default.stackApiKey !== 'string') {
+            exportQueryConfig.stackApiKey = flags['stack-api-key'] || (await (0, common_helper_1.askAPIKey)());
+            if (typeof exportQueryConfig.stackApiKey !== 'string') {
                 throw new Error('Invalid API key received');
             }
         }

package/lib/utils/index.d.ts

@@ -6,3 +6,4 @@ export * from './config-handler';
 export * from './content-type-helper';
 export * from './dependency-resolver';
 export * from './referenced-asset-handler';
+export { setupBranches } from './branch-helper';

package/lib/utils/index.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.unlinkFileLogger = exports.log = exports.fsUtil = exports.fileHelper = void 0;
+exports.setupBranches = exports.unlinkFileLogger = exports.log = exports.fsUtil = exports.fileHelper = void 0;
 const tslib_1 = require("tslib");
 exports.fileHelper = tslib_1.__importStar(require("./file-helper"));
 var file_helper_1 = require("./file-helper");
@@ -13,3 +13,5 @@ tslib_1.__exportStar(require("./config-handler"), exports);
 tslib_1.__exportStar(require("./content-type-helper"), exports);
 tslib_1.__exportStar(require("./dependency-resolver"), exports);
 tslib_1.__exportStar(require("./referenced-asset-handler"), exports);
+var branch_helper_1 = require("./branch-helper");
+Object.defineProperty(exports, "setupBranches", { enumerable: true, get: function () { return branch_helper_1.setupBranches; } });

package/lib/utils/referenced-asset-handler.js

@@ -83,10 +83,21 @@ class AssetReferenceHandler {
                 assetUIDs.add(match[1]);
             }
         }
+        let assetUrlRegex = '';
+        let assetUIDMatchIndex;
+        if (process.env.ENVIRONMENT === 'NON_PROD') {
+            assetUrlRegex = '(https://.*?/v3/assets/(.*?)/(.*?)/(.*?)/(.*?)(?="))';
+            assetUIDMatchIndex = 3;
+        }
+        else {
+            assetUrlRegex =
+                '(https://(assets|(eu-|azure-na-|azure-eu-|gcp-na-|gcp-eu-)?images).contentstack.(io|com)/v3/assets/(.*?)/(.*?)/(.*?)/(.*?)(?="))';
+            assetUIDMatchIndex = 6;
+        }
         // Pattern 2: Contentstack asset URLs
-        const urlRegex = new RegExp('(https://(assets|(eu-|azure-na-|azure-eu-|gcp-na-|gcp-eu-)?images).contentstack.(io|com)/v3/assets/(.*?)/(.*?)/(.*?)/(.*?)(?="))', 'g');
+        const urlRegex = new RegExp(assetUrlRegex, 'g');
         while ((match = urlRegex.exec(content)) !== null) {
-            const assetUID = match[6]; // The asset UID is in the 6th capture group
+            const assetUID = match[assetUIDMatchIndex]; // The asset UID is in the 6th capture group
             if (assetUID) {
                 assetUIDs.add(assetUID);
             }

package/oclif.manifest.json

@@ -1,9 +1,7 @@
 {
   "commands": {
     "cm:stacks:export-query": {
-      "aliases": [
-        "cm:export-query"
-      ],
+      "aliases": [],
       "args": {},
       "description": "Export content from a stack using query-based filtering",
       "examples": [
@@ -104,5 +102,5 @@
       ]
     }
   },
-  "version": "1.0.0-beta.1"
+  "version": "1.0.0-beta.3"
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@contentstack/cli-cm-export-query",
   "description": "Contentstack CLI plugin to export content from stack",
-  "version": "1.0.0-beta.1",
+  "version": "1.0.0-beta.3",
   "author": "Contentstack",
   "bugs": "https://github.com/contentstack/cli/issues",
   "dependencies": {