@magentrix-corp/magentrix-cli 1.3.16 → 1.3.17

package/utils/downloadAssets.js

@@ -1,292 +1,292 @@
-import { EXPORT_ROOT } from "../vars/global.js";
-import { downloadAssetsZip, listAssets } from "./magentrix/api/assets.js";
-import fs from 'fs';
-import extract from 'extract-zip';
-import { v4 as uuidv4 } from 'uuid';
-import fspath from 'path';
-import { setFileTag } from "./filetag.js";
-import { toLocalPath } from "./assetPaths.js";
-import chalk from 'chalk';
-import Config from "./config.js";
-import { sha256 } from "./hash.js";
-
-const config = new Config();
-
-export const walkAssets = async (instanceUrl, token, assetPath) => {
-    const assetResults = await listAssets(instanceUrl, token, assetPath);
-    const walkedAssets = [];
-
-    for (const asset of assetResults.Assets) {
-        if (asset.Type === 'Folder') {
-            walkedAssets.push({
-                ...asset,
-                ApiPath: asset.Path, // Keep original API path
-                Path: toLocalPath(asset.Path), // Convert for local file system
-                Children: await walkAssets(instanceUrl, token, asset.Path),
-                ParentFolder: toLocalPath(assetResults.CurrentPath),
-                ParentApiPath: assetResults.CurrentPath, // Keep API path for downloads
-            })
-        } else {
-            walkedAssets.push({
-                ...asset,
-                ApiPath: asset.Path, // Keep original API path
-                Path: toLocalPath(asset.Path), // Convert for local file system
-                ParentFolder: toLocalPath(assetResults.CurrentPath),
-                ParentApiPath: assetResults.CurrentPath, // Keep API path for downloads
-            });
-        }
-    }
-
-    return walkedAssets;
-}
-
-export const downloadAssets = async (instanceUrl, token, path, progressCallback = null, logger = null) => {
-    const allAssets = await walkAssets(instanceUrl, token, path);
-
-    // Count total files for progress tracking
-    let totalFiles = 0;
-    let downloadedFiles = 0;
-
-    const countFiles = (assets) => {
-        for (const asset of assets) {
-            if (asset.Type === 'File') {
-                totalFiles++;
-            }
-            if (asset.Type === 'Folder' && asset.Children) {
-                countFiles(asset.Children);
-            }
-        }
-    };
-    countFiles(allAssets);
-
-    // Collect all base updates to write at the end (batch operation)
-    const baseUpdates = {};
-
-    const iterateDownload = async (assets) => {
-        // Handle empty assets array
-        if (!assets || assets.length === 0) {
-            return;
-        }
-
-        const parentApiPath = assets?.[0]?.ParentApiPath; // Use API path for API calls
-        const parentLocalFolder = assets?.[0]?.ParentFolder; // Use local path for file system
-        const folders = assets.filter(asset => asset.Type === 'Folder');
-        const files = assets.filter(asset => asset.Type === 'File');
-
-        for (const folder of folders) {
-            const folderPath = fspath.join(EXPORT_ROOT, folder.Path);
-            fs.mkdirSync(folderPath, { recursive: true });
-
-            // Collect base update for folder
-            if (fs.existsSync(folderPath)) {
-                const folderStats = fs.statSync(folderPath);
-                baseUpdates[folder.Path] = {
-                    lastModified: folderStats.mtimeMs,
-                    contentHash: '',
-                    compressedContent: '',
-                    recordId: folder.Path,
-                    type: folder.Type,
-                    filePath: folder.Path,
-                    lastKnownActualPath: folderPath,
-                    lastKnownPath: fspath.resolve(folder.Path)
-                };
-            }
-
-            await iterateDownload(folder?.Children || []);
-        }
-
-        if (files.length > 0) {
-            // Batch files dynamically to avoid URL length limits
-            // Testing shows the API fails at ~2150 chars (likely 2048 limit on the server)
-            // We use 2000 as a safe limit with buffer
-            const MAX_URL_LENGTH = 2000;
-            const fileBatches = [];
-
-            // Calculate base URL length once (everything except the file names)
-            const baseUrl = new URL("/api/3.0/staticassets/download", instanceUrl);
-            baseUrl.searchParams.set("path", parentApiPath);
-            baseUrl.searchParams.set('download-format', 'zip');
-            // The names will be added as: &names=encoded_comma_separated_list
-            // Calculate the base without names parameter
-            const baseUrlStr = baseUrl.toString();
-            const baseLength = baseUrlStr.length + '&names='.length;
-
-            let currentBatch = [];
-            let currentNames = '';
-
-            for (const file of files) {
-                // Calculate what the names parameter would be if we add this file
-                const testNames = currentNames
-                    ? `${currentNames},${file.Name}`
-                    : file.Name;
-
-                // Calculate the URL length with this names string encoded
-                const encodedTestNames = encodeURIComponent(testNames);
-                const testLength = baseLength + encodedTestNames.length;
-
-                // Check if adding this file would exceed the URL limit
-                if (testLength > MAX_URL_LENGTH && currentBatch.length > 0) {
-                    // Start a new batch - current batch is full
-                    fileBatches.push([...currentBatch]);
-                    currentBatch = [file];
-                    currentNames = file.Name;
-                } else {
-                    // Add to current batch
-                    currentBatch.push(file);
-                    currentNames = testNames;
-                }
-            }
-
-            // Add the last batch if it has files
-            if (currentBatch.length > 0) {
-                fileBatches.push(currentBatch);
-            }
-
-            for (let batchIndex = 0; batchIndex < fileBatches.length; batchIndex++) {
-                const batch = fileBatches[batchIndex];
-
-                // Skip empty batches
-                if (!batch || batch.length === 0) {
-                    continue;
-                }
-
-                const batchZipName = fileBatches.length > 1
-                    ? `assets-batch-${batchIndex + 1}.zip`
-                    : 'assets.zip';
-
-                try {
-                    // Validate that we have the required data
-                    if (!parentApiPath) {
-                        console.warn(`Warning: Skipping batch because parentApiPath is undefined. Files: ${batch.map(f => f.Name).join(', ')}`);
-                        continue;
-                    }
-
-                    if (!parentLocalFolder) {
-                        console.warn(`Warning: Skipping batch because parentLocalFolder is undefined. Path: ${parentApiPath}`);
-                        continue;
-                    }
-
-                    // Debug logging for problematic paths
-                    if (logger && parentApiPath.includes('@')) {
-                        logger.info('Downloading batch with special characters', {
-                            path: parentApiPath,
-                            fileCount: batch.length,
-                            firstFile: batch[0]?.Name
-                        });
-                    }
-
-                    const savedAs = await downloadAssetsZip({
-                        baseUrl: instanceUrl,
-                        token: token, // "Bearer" prefix added in code
-                        path: parentApiPath, // Use API path for API call
-                        names: batch.map(file => file.Name),
-                        outFile: fspath.join(EXPORT_ROOT, parentLocalFolder, batchZipName), // Use local path for file system
-                    });
-
-                    await extract(savedAs, {
-                        dir: fspath.resolve(fspath.join(EXPORT_ROOT, parentLocalFolder)) // Use local path for extraction
-                    });
-
-                    fs.rmSync(savedAs);
-
-                    // Collect base updates for all files in this batch
-                    // We'll write them all at once at the end for performance
-                    for (const file of batch) {
-                        try {
-                            const filePath = fspath.join(EXPORT_ROOT, file.Path);
-                            if (fs.existsSync(filePath)) {
-                                const fileStats = fs.statSync(filePath);
-                                const fileContent = fs.readFileSync(filePath, "utf-8");
-                                const contentHash = sha256(fileContent);
-
-                                baseUpdates[file.Path] = {
-                                    lastModified: fileStats.mtimeMs,
-                                    contentHash,
-                                    compressedContent: '', // Assets don't store content
-                                    recordId: file.Path,
-                                    type: file.Type,
-                                    filePath: file.Path,
-                                    lastKnownActualPath: filePath,
-                                    lastKnownPath: fspath.resolve(file.Path)
-                                };
-                            }
-                        } catch (err) {
-                            // Log but don't fail the download
-                            if (logger) {
-                                logger.warning(`Failed to prepare base update for ${file.Path}`, { error: err.message });
-                            }
-                        }
-                    }
-
-                    // Update progress
-                    downloadedFiles += batch.length;
-                    if (progressCallback) {
-                        progressCallback(downloadedFiles, totalFiles, `Downloaded ${downloadedFiles}/${totalFiles} files`);
-                    }
-                } catch (error) {
-                    // Check if this is a 404 error (files don't exist on server)
-                    const is404 = error.message && error.message.includes('404');
-
-                    if (is404) {
-                        // Files don't exist on server - log info but continue
-                        // This is expected behavior for files that were deleted on the server
-                        const fileNames = batch.map(f => f.Name).join(', ');
-                        const infoMessage = `Skipped ${batch.length} missing files from ${parentApiPath || '(undefined path)'}`;
-
-                        console.warn(chalk.gray(`\n  ℹ️  ${infoMessage}`));
-                        console.warn(chalk.gray(`     These files don't exist on the server anymore.`));
-
-                        // Log to file if logger is available (as INFO, not WARNING)
-                        if (logger) {
-                            logger.info(infoMessage, {
-                                path: parentApiPath,
-                                fileCount: batch.length,
-                                firstFewFiles: fileNames.substring(0, 200) + (fileNames.length > 200 ? '...' : '')
-                            });
-                        }
-
-                        // Still update progress as if we "downloaded" them (they don't exist)
-                        downloadedFiles += batch.length;
-                        if (progressCallback) {
-                            progressCallback(downloadedFiles, totalFiles, `Skipped ${batch.length} missing files`);
-                        }
-                    } else {
-                        // Other errors should still fail
-                        const fileNames = batch.map(f => f.Name).join(', ');
-                        console.error(`\nFailed to download batch from ${parentApiPath || '(undefined path)'}`);
-                        console.error(`Files: ${fileNames}`);
-                        console.error(`Error: ${error.message}\n`);
-                        throw error;
-                    }
-                }
-            }
-        }
-    }
-
-    await iterateDownload(allAssets);
-
-    // Write all base updates in ONE operation at the end (super fast!)
-    if (Object.keys(baseUpdates).length > 0) {
-        if (logger) {
-            logger.info(`Writing ${Object.keys(baseUpdates).length} asset base updates to base.json`);
-        }
-
-        // Read current base.json once (pass null as key to get entire config)
-        const baseConfig = config.read(null, { filename: "base.json" }) || {};
-
-        // Merge all updates
-        Object.assign(baseConfig, baseUpdates);
-
-        // Write once
-        const baseJsonPath = fspath.join(fspath.dirname(config.projectConfigPath), "base.json");
-        fs.writeFileSync(baseJsonPath, JSON.stringify(baseConfig, null, 2), { mode: 0o600 });
-
-        if (logger) {
-            logger.info(`Successfully wrote all asset base updates`);
-        }
-    }
-
-    return {
-        tree: allAssets
-    };
+import { EXPORT_ROOT } from "../vars/global.js";
+import { downloadAssetsZip, listAssets } from "./magentrix/api/assets.js";
+import fs from 'fs';
+import extract from 'extract-zip';
+import { v4 as uuidv4 } from 'uuid';
+import fspath from 'path';
+import { setFileTag } from "./filetag.js";
+import { toLocalPath } from "./assetPaths.js";
+import chalk from 'chalk';
+import Config from "./config.js";
+import { sha256 } from "./hash.js";
+
+const config = new Config();
+
+export const walkAssets = async (instanceUrl, token, assetPath) => {
+    const assetResults = await listAssets(instanceUrl, token, assetPath);
+    const walkedAssets = [];
+
+    for (const asset of assetResults.Assets) {
+        if (asset.Type === 'Folder') {
+            walkedAssets.push({
+                ...asset,
+                ApiPath: asset.Path, // Keep original API path
+                Path: toLocalPath(asset.Path), // Convert for local file system
+                Children: await walkAssets(instanceUrl, token, asset.Path),
+                ParentFolder: toLocalPath(assetResults.CurrentPath),
+                ParentApiPath: assetResults.CurrentPath, // Keep API path for downloads
+            })
+        } else {
+            walkedAssets.push({
+                ...asset,
+                ApiPath: asset.Path, // Keep original API path
+                Path: toLocalPath(asset.Path), // Convert for local file system
+                ParentFolder: toLocalPath(assetResults.CurrentPath),
+                ParentApiPath: assetResults.CurrentPath, // Keep API path for downloads
+            });
+        }
+    }
+
+    return walkedAssets;
+}
+
+export const downloadAssets = async (instanceUrl, token, path, progressCallback = null, logger = null) => {
+    const allAssets = await walkAssets(instanceUrl, token, path);
+
+    // Count total files for progress tracking
+    let totalFiles = 0;
+    let downloadedFiles = 0;
+
+    const countFiles = (assets) => {
+        for (const asset of assets) {
+            if (asset.Type === 'File') {
+                totalFiles++;
+            }
+            if (asset.Type === 'Folder' && asset.Children) {
+                countFiles(asset.Children);
+            }
+        }
+    };
+    countFiles(allAssets);
+
+    // Collect all base updates to write at the end (batch operation)
+    const baseUpdates = {};
+
+    const iterateDownload = async (assets) => {
+        // Handle empty assets array
+        if (!assets || assets.length === 0) {
+            return;
+        }
+
+        const parentApiPath = assets?.[0]?.ParentApiPath; // Use API path for API calls
+        const parentLocalFolder = assets?.[0]?.ParentFolder; // Use local path for file system
+        const folders = assets.filter(asset => asset.Type === 'Folder');
+        const files = assets.filter(asset => asset.Type === 'File');
+
+        for (const folder of folders) {
+            const folderPath = fspath.join(EXPORT_ROOT, folder.Path);
+            fs.mkdirSync(folderPath, { recursive: true });
+
+            // Collect base update for folder
+            if (fs.existsSync(folderPath)) {
+                const folderStats = fs.statSync(folderPath);
+                baseUpdates[folder.Path] = {
+                    lastModified: folderStats.mtimeMs,
+                    contentHash: '',
+                    compressedContent: '',
+                    recordId: folder.Path,
+                    type: folder.Type,
+                    filePath: folder.Path,
+                    lastKnownActualPath: folderPath,
+                    lastKnownPath: fspath.resolve(folder.Path)
+                };
+            }
+
+            await iterateDownload(folder?.Children || []);
+        }
+
+        if (files.length > 0) {
+            // Batch files dynamically to avoid URL length limits
+            // Testing shows the API fails at ~2150 chars (likely 2048 limit on the server)
+            // We use 2000 as a safe limit with buffer
+            const MAX_URL_LENGTH = 2000;
+            const fileBatches = [];
+
+            // Calculate base URL length once (everything except the file names)
+            const baseUrl = new URL("/api/3.0/staticassets/download", instanceUrl);
+            baseUrl.searchParams.set("path", parentApiPath);
+            baseUrl.searchParams.set('download-format', 'zip');
+            // The names will be added as: &names=encoded_comma_separated_list
+            // Calculate the base without names parameter
+            const baseUrlStr = baseUrl.toString();
+            const baseLength = baseUrlStr.length + '&names='.length;
+
+            let currentBatch = [];
+            let currentNames = '';
+
+            for (const file of files) {
+                // Calculate what the names parameter would be if we add this file
+                const testNames = currentNames
+                    ? `${currentNames},${file.Name}`
+                    : file.Name;
+
+                // Calculate the URL length with this names string encoded
+                const encodedTestNames = encodeURIComponent(testNames);
+                const testLength = baseLength + encodedTestNames.length;
+
+                // Check if adding this file would exceed the URL limit
+                if (testLength > MAX_URL_LENGTH && currentBatch.length > 0) {
+                    // Start a new batch - current batch is full
+                    fileBatches.push([...currentBatch]);
+                    currentBatch = [file];
+                    currentNames = file.Name;
+                } else {
+                    // Add to current batch
+                    currentBatch.push(file);
+                    currentNames = testNames;
+                }
+            }
+
+            // Add the last batch if it has files
+            if (currentBatch.length > 0) {
+                fileBatches.push(currentBatch);
+            }
+
+            for (let batchIndex = 0; batchIndex < fileBatches.length; batchIndex++) {
+                const batch = fileBatches[batchIndex];
+
+                // Skip empty batches
+                if (!batch || batch.length === 0) {
+                    continue;
+                }
+
+                const batchZipName = fileBatches.length > 1
+                    ? `assets-batch-${batchIndex + 1}.zip`
+                    : 'assets.zip';
+
+                try {
+                    // Validate that we have the required data
+                    if (!parentApiPath) {
+                        console.warn(`Warning: Skipping batch because parentApiPath is undefined. Files: ${batch.map(f => f.Name).join(', ')}`);
+                        continue;
+                    }
+
+                    if (!parentLocalFolder) {
+                        console.warn(`Warning: Skipping batch because parentLocalFolder is undefined. Path: ${parentApiPath}`);
+                        continue;
+                    }
+
+                    // Debug logging for problematic paths
+                    if (logger && parentApiPath.includes('@')) {
+                        logger.info('Downloading batch with special characters', {
+                            path: parentApiPath,
+                            fileCount: batch.length,
+                            firstFile: batch[0]?.Name
+                        });
+                    }
+
+                    const savedAs = await downloadAssetsZip({
+                        baseUrl: instanceUrl,
+                        token: token, // "Bearer" prefix added in code
+                        path: parentApiPath, // Use API path for API call
+                        names: batch.map(file => file.Name),
+                        outFile: fspath.join(EXPORT_ROOT, parentLocalFolder, batchZipName), // Use local path for file system
+                    });
+
+                    await extract(savedAs, {
+                        dir: fspath.resolve(fspath.join(EXPORT_ROOT, parentLocalFolder)) // Use local path for extraction
+                    });
+
+                    fs.rmSync(savedAs);
+
+                    // Collect base updates for all files in this batch
+                    // We'll write them all at once at the end for performance
+                    for (const file of batch) {
+                        try {
+                            const filePath = fspath.join(EXPORT_ROOT, file.Path);
+                            if (fs.existsSync(filePath)) {
+                                const fileStats = fs.statSync(filePath);
+                                const fileContent = fs.readFileSync(filePath, "utf-8");
+                                const contentHash = sha256(fileContent);
+
+                                baseUpdates[file.Path] = {
+                                    lastModified: fileStats.mtimeMs,
+                                    contentHash,
+                                    compressedContent: '', // Assets don't store content
+                                    recordId: file.Path,
+                                    type: file.Type,
+                                    filePath: file.Path,
+                                    lastKnownActualPath: filePath,
+                                    lastKnownPath: fspath.resolve(file.Path)
+                                };
+                            }
+                        } catch (err) {
+                            // Log but don't fail the download
+                            if (logger) {
+                                logger.warning(`Failed to prepare base update for ${file.Path}`, { error: err.message });
+                            }
+                        }
+                    }
+
+                    // Update progress
+                    downloadedFiles += batch.length;
+                    if (progressCallback) {
+                        progressCallback(downloadedFiles, totalFiles, `Downloaded ${downloadedFiles}/${totalFiles} files`);
+                    }
+                } catch (error) {
+                    // Check if this is a 404 error (files don't exist on server)
+                    const is404 = error.message && error.message.includes('404');
+
+                    if (is404) {
+                        // Files don't exist on server - log info but continue
+                        // This is expected behavior for files that were deleted on the server
+                        const fileNames = batch.map(f => f.Name).join(', ');
+                        const infoMessage = `Skipped ${batch.length} missing files from ${parentApiPath || '(undefined path)'}`;
+
+                        console.warn(chalk.gray(`\n  ℹ️  ${infoMessage}`));
+                        console.warn(chalk.gray(`     These files don't exist on the server anymore.`));
+
+                        // Log to file if logger is available (as INFO, not WARNING)
+                        if (logger) {
+                            logger.info(infoMessage, {
+                                path: parentApiPath,
+                                fileCount: batch.length,
+                                firstFewFiles: fileNames.substring(0, 200) + (fileNames.length > 200 ? '...' : '')
+                            });
+                        }
+
+                        // Still update progress as if we "downloaded" them (they don't exist)
+                        downloadedFiles += batch.length;
+                        if (progressCallback) {
+                            progressCallback(downloadedFiles, totalFiles, `Skipped ${batch.length} missing files`);
+                        }
+                    } else {
+                        // Other errors should still fail
+                        const fileNames = batch.map(f => f.Name).join(', ');
+                        console.error(`\nFailed to download batch from ${parentApiPath || '(undefined path)'}`);
+                        console.error(`Files: ${fileNames}`);
+                        console.error(`Error: ${error.message}\n`);
+                        throw error;
+                    }
+                }
+            }
+        }
+    }
+
+    await iterateDownload(allAssets);
+
+    // Write all base updates in ONE operation at the end (super fast!)
+    if (Object.keys(baseUpdates).length > 0) {
+        if (logger) {
+            logger.info(`Writing ${Object.keys(baseUpdates).length} asset base updates to base.json`);
+        }
+
+        // Read current base.json once (pass null as key to get entire config)
+        const baseConfig = config.read(null, { filename: "base.json" }) || {};
+
+        // Merge all updates
+        Object.assign(baseConfig, baseUpdates);
+
+        // Write once
+        const baseJsonPath = fspath.join(fspath.dirname(config.projectConfigPath), "base.json");
+        fs.writeFileSync(baseJsonPath, JSON.stringify(baseConfig, null, 2), { mode: 0o600 });
+
+        if (logger) {
+            logger.info(`Successfully wrote all asset base updates`);
+        }
+    }
+
+    return {
+        tree: allAssets
+    };
 }