@magentrix-corp/magentrix-cli 1.1.4 → 1.2.0

package/utils/diagnostics/testPublishLogic.js

@@ -0,0 +1,96 @@
+import { walkFiles } from '../cacher.js';
+import Config from '../config.js';
+import path from 'path';
+
+const config = new Config();
+const EXPORT_ROOT = 'src';
+
+// Load cache
+const hits = await config.searchObject({}, { filename: "base.json", global: false });
+const cachedResults = hits?.[0]?.value || {};
+
+const cachedFiles = Object.values(cachedResults).map((c) => ({
+    ...c,
+    tag: c.recordId,
+    filePath: c.filePath || c.lastKnownPath,
+}));
+
+console.log('Total cached files:', cachedFiles.length);
+console.log('Cached assets (type=File):', cachedFiles.filter(cf => cf.type === 'File').length);
+console.log('Cached folders (type=Folder):', cachedFiles.filter(cf => cf.type === 'Folder').length);
+console.log('');
+
+// Build Set like in publish
+const cachedAssetPaths = new Set();
+cachedFiles
+    .filter(cf => cf.type === 'File' || cf.type === 'Folder')
+    .forEach(cf => {
+        if (cf.lastKnownActualPath) {
+            cachedAssetPaths.add(path.normalize(path.resolve(cf.lastKnownActualPath)).toLowerCase());
+        }
+        if (cf.filePath) {
+            cachedAssetPaths.add(path.normalize(path.resolve(cf.filePath)).toLowerCase());
+        }
+        if (cf.lastKnownPath) {
+            cachedAssetPaths.add(path.normalize(path.resolve(cf.lastKnownPath)).toLowerCase());
+        }
+    });
+
+console.log('Cached asset paths in Set:', cachedAssetPaths.size);
+console.log('');
+
+// Get local assets
+const assetPaths = await walkFiles(path.join(EXPORT_ROOT, 'Assets'));
+console.log('Local asset files found:', assetPaths.length);
+console.log('');
+
+// Check first few
+let matched = 0;
+let notMatched = 0;
+
+for (let i = 0; i < Math.min(10, assetPaths.length); i++) {
+    const assetPath = assetPaths[i];
+    const normalizedAssetPath = path.normalize(path.resolve(assetPath)).toLowerCase();
+    const inCache = cachedAssetPaths.has(normalizedAssetPath);
+
+    if (inCache) {
+        matched++;
+    } else {
+        notMatched++;
+        console.log('NOT IN CACHE:', assetPath);
+        console.log('  Normalized:', normalizedAssetPath);
+        console.log('');
+    }
+}
+
+console.log(`First 10 files: ${matched} matched, ${notMatched} not matched`);
+console.log('');
+
+// Check if the Set actually contains the sample path
+const samplePath = assetPaths[0];
+const sampleNormalized = path.normalize(path.resolve(samplePath)).toLowerCase();
+console.log('Sample path:', samplePath);
+console.log('Normalized:', sampleNormalized);
+console.log('In Set:', cachedAssetPaths.has(sampleNormalized));
+console.log('');
+
+// Check what's actually in the Set for this file
+const sampleInBase = Object.values(cachedResults).find(b =>
+    b.filePath === 'Assets/Acronis/Banners/1.png' ||
+    b.lastKnownActualPath === 'src/Assets/Acronis/Banners/1.png'
+);
+
+if (sampleInBase) {
+    console.log('Found in base.json:');
+    console.log('  type:', sampleInBase.type);
+    console.log('  filePath:', sampleInBase.filePath);
+    console.log('  lastKnownActualPath:', sampleInBase.lastKnownActualPath);
+    console.log('  lastKnownPath:', sampleInBase.lastKnownPath);
+
+    // Check what gets added to the Set
+    if (sampleInBase.lastKnownActualPath) {
+        const normalized = path.normalize(path.resolve(sampleInBase.lastKnownActualPath)).toLowerCase();
+        console.log('  Normalized lastKnownActualPath:', normalized);
+        console.log('  Match:', normalized === sampleNormalized);
+    }
+}

package/utils/downloadAssets.js

@@ -6,6 +6,11 @@ import { v4 as uuidv4 } from 'uuid';
 import fspath from 'path';
 import { setFileTag } from "./filetag.js";
 import { toLocalPath } from "./assetPaths.js";
+import chalk from 'chalk';
+import Config from "./config.js";
+import { sha256 } from "./hash.js";
+
+const config = new Config();
 
 export const walkAssets = async (instanceUrl, token, assetPath) => {
     const assetResults = await listAssets(instanceUrl, token, assetPath);
@@ -35,46 +40,252 @@ export const walkAssets = async (instanceUrl, token, assetPath) => {
     return walkedAssets;
 }
 
-export const downloadAssets = async (instanceUrl, token, path) => {
+export const downloadAssets = async (instanceUrl, token, path, progressCallback = null, logger = null) => {
     const allAssets = await walkAssets(instanceUrl, token, path);
 
+    // Count total files for progress tracking
+    let totalFiles = 0;
+    let downloadedFiles = 0;
+
+    const countFiles = (assets) => {
+        for (const asset of assets) {
+            if (asset.Type === 'File') {
+                totalFiles++;
+            }
+            if (asset.Type === 'Folder' && asset.Children) {
+                countFiles(asset.Children);
+            }
+        }
+    };
+    countFiles(allAssets);
+
+    // Collect all base updates to write at the end (batch operation)
+    const baseUpdates = {};
+
     const iterateDownload = async (assets) => {
+        // Handle empty assets array
+        if (!assets || assets.length === 0) {
+            return;
+        }
+
         const parentApiPath = assets?.[0]?.ParentApiPath; // Use API path for API calls
         const parentLocalFolder = assets?.[0]?.ParentFolder; // Use local path for file system
         const folders = assets.filter(asset => asset.Type === 'Folder');
         const files = assets.filter(asset => asset.Type === 'File');
 
         for (const folder of folders) {
-            fs.mkdirSync(fspath.join(EXPORT_ROOT, folder.Path), { recursive: true });
+            const folderPath = fspath.join(EXPORT_ROOT, folder.Path);
+            fs.mkdirSync(folderPath, { recursive: true });
+
+            // Collect base update for folder
+            if (fs.existsSync(folderPath)) {
+                const folderStats = fs.statSync(folderPath);
+                baseUpdates[folder.Path] = {
+                    lastModified: folderStats.mtimeMs,
+                    contentHash: '',
+                    compressedContent: '',
+                    recordId: folder.Path,
+                    type: folder.Type,
+                    filePath: folder.Path,
+                    lastKnownActualPath: folderPath,
+                    lastKnownPath: fspath.resolve(folder.Path)
+                };
+            }
+
             await iterateDownload(folder?.Children || []);
         }
 
         if (files.length > 0) {
-            const savedAs = await downloadAssetsZip({
-                baseUrl: instanceUrl,
-                token: token, // "Bearer" prefix added in code
-                path: parentApiPath, // Use API path for API call
-                names: files.map(file => file.Name),
-                outFile: fspath.join(EXPORT_ROOT, parentLocalFolder, 'assets.zip'), // Use local path for file system
-            });
+            // Batch files dynamically to avoid URL length limits
+            // Testing shows the API fails at ~2150 chars (likely 2048 limit on the server)
+            // We use 2000 as a safe limit with buffer
+            const MAX_URL_LENGTH = 2000;
+            const fileBatches = [];
 
-            await extract(savedAs, {
-                dir: fspath.resolve(fspath.join(EXPORT_ROOT, parentLocalFolder)) // Use local path for extraction
-            });
+            // Calculate base URL length once (everything except the file names)
+            const baseUrl = new URL("/api/3.0/staticassets/download", instanceUrl);
+            baseUrl.searchParams.set("path", parentApiPath);
+            baseUrl.searchParams.set('download-format', 'zip');
+            // The names will be added as: &names=encoded_comma_separated_list
+            // Calculate the base without names parameter
+            const baseUrlStr = baseUrl.toString();
+            const baseLength = baseUrlStr.length + '&names='.length;
+
+            let currentBatch = [];
+            let currentNames = '';
+
+            for (const file of files) {
+                // Calculate what the names parameter would be if we add this file
+                const testNames = currentNames
+                    ? `${currentNames},${file.Name}`
+                    : file.Name;
+
+                // Calculate the URL length with this names string encoded
+                const encodedTestNames = encodeURIComponent(testNames);
+                const testLength = baseLength + encodedTestNames.length;
+
+                // Check if adding this file would exceed the URL limit
+                if (testLength > MAX_URL_LENGTH && currentBatch.length > 0) {
+                    // Start a new batch - current batch is full
+                    fileBatches.push([...currentBatch]);
+                    currentBatch = [file];
+                    currentNames = file.Name;
+                } else {
+                    // Add to current batch
+                    currentBatch.push(file);
+                    currentNames = testNames;
+                }
+            }
+
+            // Add the last batch if it has files
+            if (currentBatch.length > 0) {
+                fileBatches.push(currentBatch);
+            }
+
+            for (let batchIndex = 0; batchIndex < fileBatches.length; batchIndex++) {
+                const batch = fileBatches[batchIndex];
+
+                // Skip empty batches
+                if (!batch || batch.length === 0) {
+                    continue;
+                }
+
+                const batchZipName = fileBatches.length > 1
+                    ? `assets-batch-${batchIndex + 1}.zip`
+                    : 'assets.zip';
 
-            // for (const file of files) {
-            //     // Ensure the file was created properly
-            //     // TODO: Create some error handling incase it wasn't
-            //     const downloadedPath = fspath.join(EXPORT_ROOT, file.Path);
-            //     if (!fs.existsSync(downloadedPath)) continue;
-            // }
+                try {
+                    // Validate that we have the required data
+                    if (!parentApiPath) {
+                        console.warn(`Warning: Skipping batch because parentApiPath is undefined. Files: ${batch.map(f => f.Name).join(', ')}`);
+                        continue;
+                    }
 
-            fs.rmSync(savedAs);
+                    if (!parentLocalFolder) {
+                        console.warn(`Warning: Skipping batch because parentLocalFolder is undefined. Path: ${parentApiPath}`);
+                        continue;
+                    }
+
+                    // Debug logging for problematic paths
+                    if (logger && parentApiPath.includes('@')) {
+                        logger.info('Downloading batch with special characters', {
+                            path: parentApiPath,
+                            fileCount: batch.length,
+                            firstFile: batch[0]?.Name
+                        });
+                    }
+
+                    const savedAs = await downloadAssetsZip({
+                        baseUrl: instanceUrl,
+                        token: token, // "Bearer" prefix added in code
+                        path: parentApiPath, // Use API path for API call
+                        names: batch.map(file => file.Name),
+                        outFile: fspath.join(EXPORT_ROOT, parentLocalFolder, batchZipName), // Use local path for file system
+                    });
+
+                    await extract(savedAs, {
+                        dir: fspath.resolve(fspath.join(EXPORT_ROOT, parentLocalFolder)) // Use local path for extraction
+                    });
+
+                    fs.rmSync(savedAs);
+
+                    // Collect base updates for all files in this batch
+                    // We'll write them all at once at the end for performance
+                    for (const file of batch) {
+                        try {
+                            const filePath = fspath.join(EXPORT_ROOT, file.Path);
+                            if (fs.existsSync(filePath)) {
+                                const fileStats = fs.statSync(filePath);
+                                const fileContent = fs.readFileSync(filePath, "utf-8");
+                                const contentHash = sha256(fileContent);
+
+                                baseUpdates[file.Path] = {
+                                    lastModified: fileStats.mtimeMs,
+                                    contentHash,
+                                    compressedContent: '', // Assets don't store content
+                                    recordId: file.Path,
+                                    type: file.Type,
+                                    filePath: file.Path,
+                                    lastKnownActualPath: filePath,
+                                    lastKnownPath: fspath.resolve(file.Path)
+                                };
+                            }
+                        } catch (err) {
+                            // Log but don't fail the download
+                            if (logger) {
+                                logger.warning(`Failed to prepare base update for ${file.Path}`, { error: err.message });
+                            }
+                        }
+                    }
+
+                    // Update progress
+                    downloadedFiles += batch.length;
+                    if (progressCallback) {
+                        progressCallback(downloadedFiles, totalFiles, `Downloaded ${downloadedFiles}/${totalFiles} files`);
+                    }
+                } catch (error) {
+                    // Check if this is a 404 error (files don't exist on server)
+                    const is404 = error.message && error.message.includes('404');
+
+                    if (is404) {
+                        // Files don't exist on server - log info but continue
+                        // This is expected behavior for files that were deleted on the server
+                        const fileNames = batch.map(f => f.Name).join(', ');
+                        const infoMessage = `Skipped ${batch.length} missing files from ${parentApiPath || '(undefined path)'}`;
+
+                        console.warn(chalk.gray(`\n  ℹ️  ${infoMessage}`));
+                        console.warn(chalk.gray(`     These files don't exist on the server anymore.`));
+
+                        // Log to file if logger is available (as INFO, not WARNING)
+                        if (logger) {
+                            logger.info(infoMessage, {
+                                path: parentApiPath,
+                                fileCount: batch.length,
+                                firstFewFiles: fileNames.substring(0, 200) + (fileNames.length > 200 ? '...' : '')
+                            });
+                        }
+
+                        // Still update progress as if we "downloaded" them (they don't exist)
+                        downloadedFiles += batch.length;
+                        if (progressCallback) {
+                            progressCallback(downloadedFiles, totalFiles, `Skipped ${batch.length} missing files`);
+                        }
+                    } else {
+                        // Other errors should still fail
+                        const fileNames = batch.map(f => f.Name).join(', ');
+                        console.error(`\nFailed to download batch from ${parentApiPath || '(undefined path)'}`);
+                        console.error(`Files: ${fileNames}`);
+                        console.error(`Error: ${error.message}\n`);
+                        throw error;
+                    }
+                }
+            }
         }
     }
 
     await iterateDownload(allAssets);
 
+    // Write all base updates in ONE operation at the end (super fast!)
+    if (Object.keys(baseUpdates).length > 0) {
+        if (logger) {
+            logger.info(`Writing ${Object.keys(baseUpdates).length} asset base updates to base.json`);
+        }
+
+        // Read current base.json once (pass null as key to get entire config)
+        const baseConfig = config.read(null, { filename: "base.json" }) || {};
+
+        // Merge all updates
+        Object.assign(baseConfig, baseUpdates);
+
+        // Write once
+        const baseJsonPath = fspath.join(fspath.dirname(config.projectConfigPath), "base.json");
+        fs.writeFileSync(baseJsonPath, JSON.stringify(baseConfig, null, 2), { mode: 0o600 });
+
+        if (logger) {
+            logger.info(`Successfully wrote all asset base updates`);
+        }
+    }
+
     return {
         tree: allAssets
     };

package/utils/logger.js

@@ -0,0 +1,283 @@
+import fs from 'fs';
+import path from 'path';
+import chalk from 'chalk';
+import Config from './config.js';
+import inquirer from 'inquirer';
+
+const config = new Config();
+
+/**
+ * Logger utility for tracking errors, warnings, and operations during pull/publish.
+ * Logs are stored in .magentrix/logs/ directory.
+ */
+
+export class Logger {
+    constructor(operationName = 'operation') {
+        this.operationName = operationName;
+        this.logDir = '.magentrix/logs';
+        this.timestamp = new Date().toISOString().replace(/[:.]/g, '-').split('.')[0];
+        this.logFile = path.join(this.logDir, `${operationName}-${this.timestamp}.log`);
+        this.errors = [];
+        this.warnings = [];
+        this.infos = [];
+        this.loggingEnabled = null; // Will be determined on first write
+
+        // Check if logging is enabled in config
+        this.checkLoggingPreference();
+    }
+
+    /**
+     * Check user's logging preference from config
+     */
+    checkLoggingPreference() {
+        const preference = config.read('saveLogs', { global: true });
+
+        if (preference !== undefined && preference !== null) {
+            this.loggingEnabled = preference === true || preference === 'true';
+        }
+        // If undefined, we'll prompt on first write
+    }
+
+    /**
+     * Prompt user about logging preference (only once)
+     */
+    async promptLoggingPreference() {
+        console.log('');
+        console.log(chalk.cyan('📋 Log File Settings'));
+        console.log(chalk.gray('Magentrix CLI can save detailed operation logs for debugging.'));
+        console.log('');
+
+        const answer = await inquirer.prompt([
+            {
+                type: 'confirm',
+                name: 'saveLogs',
+                message: 'Would you like to save operation logs to files?',
+                default: true
+            }
+        ]);
+
+        this.loggingEnabled = answer.saveLogs;
+        config.save('saveLogs', answer.saveLogs, { global: true });
+
+        if (answer.saveLogs) {
+            console.log(chalk.green('✓ Logs will be saved to .magentrix/logs/'));
+        } else {
+            console.log(chalk.gray('Logs disabled. You can enable them anytime with: magentrix config set logs true'));
+        }
+        console.log('');
+    }
+
+    /**
+     * Initialize log file if logging is enabled
+     */
+    async initLogFile() {
+        if (this.loggingEnabled === null) {
+            // First time - prompt user
+            await this.promptLoggingPreference();
+        }
+
+        if (this.loggingEnabled) {
+            // Ensure log directory exists
+            if (!fs.existsSync(this.logDir)) {
+                fs.mkdirSync(this.logDir, { recursive: true });
+            }
+
+            // Initialize log file
+            this.writeToFile(`=== ${this.operationName.toUpperCase()} LOG ===`);
+            this.writeToFile(`Started: ${new Date().toISOString()}`);
+            this.writeToFile(`Working Directory: ${process.cwd()}`);
+            this.writeToFile('');
+        }
+    }
+
+    /**
+     * Write a line to the log file (internal method)
+     */
+    writeToFile(message) {
+        if (this.loggingEnabled) {
+            fs.appendFileSync(this.logFile, message + '\n');
+        }
+    }
+
+    /**
+     * Log an info message
+     */
+    info(message, details = null) {
+        const timestamp = new Date().toISOString();
+        const entry = { timestamp, level: 'INFO', message, details };
+        this.infos.push(entry);
+
+        this.writeToFile(`[${timestamp}] INFO: ${message}`);
+        if (details) {
+            this.writeToFile(`  Details: ${typeof details === 'object' ? JSON.stringify(details, null, 2) : details}`);
+        }
+    }
+
+    /**
+     * Log a warning message
+     */
+    warning(message, details = null) {
+        const timestamp = new Date().toISOString();
+        const entry = { timestamp, level: 'WARNING', message, details };
+        this.warnings.push(entry);
+
+        this.writeToFile(`[${timestamp}] WARNING: ${message}`);
+        if (details) {
+            this.writeToFile(`  Details: ${typeof details === 'object' ? JSON.stringify(details, null, 2) : details}`);
+        }
+    }
+
+    /**
+     * Log an error message
+     */
+    error(message, error = null, details = null) {
+        const timestamp = new Date().toISOString();
+        const entry = {
+            timestamp,
+            level: 'ERROR',
+            message,
+            error: error ? error.toString() : null,
+            stack: error?.stack || null,
+            details
+        };
+        this.errors.push(entry);
+
+        this.writeToFile(`[${timestamp}] ERROR: ${message}`);
+        if (error) {
+            this.writeToFile(`  Error: ${error.toString()}`);
+            if (error.stack) {
+                this.writeToFile(`  Stack: ${error.stack}`);
+            }
+        }
+        if (details) {
+            this.writeToFile(`  Details: ${typeof details === 'object' ? JSON.stringify(details, null, 2) : details}`);
+        }
+    }
+
+    /**
+     * Get a summary of the log
+     */
+    getSummary() {
+        return {
+            errors: this.errors.length,
+            warnings: this.warnings.length,
+            infos: this.infos.length,
+            logFile: this.logFile
+        };
+    }
+
+    /**
+     * Get the most recent errors (for preview)
+     */
+    getRecentErrors(count = 3) {
+        return this.errors.slice(-count);
+    }
+
+    /**
+     * Get the most recent warnings (for preview)
+     */
+    getRecentWarnings(count = 3) {
+        return this.warnings.slice(-count);
+    }
+
+    /**
+     * Display a summary to the console
+     */
+    displaySummary() {
+        const summary = this.getSummary();
+
+        if (summary.errors > 0 || summary.warnings > 0) {
+            console.log('');
+            console.log(chalk.bold('Issues Summary:'));
+
+            if (summary.errors > 0) {
+                console.log(chalk.red(`  ✗ ${summary.errors} error(s)`));
+
+                // Show preview of recent errors
+                const recentErrors = this.getRecentErrors(2);
+                recentErrors.forEach((err, idx) => {
+                    console.log(chalk.red(`    ${idx + 1}. ${err.message}`));
+                });
+
+                if (summary.errors > 2) {
+                    console.log(chalk.gray(`    ... and ${summary.errors - 2} more`));
+                }
+            }
+
+            if (summary.warnings > 0) {
+                console.log(chalk.yellow(`  ⚠ ${summary.warnings} warning(s)`));
+
+                // Show preview of recent warnings
+                const recentWarnings = this.getRecentWarnings(2);
+                recentWarnings.forEach((warn, idx) => {
+                    console.log(chalk.yellow(`    ${idx + 1}. ${warn.message}`));
+                });
+
+                if (summary.warnings > 2) {
+                    console.log(chalk.gray(`    ... and ${summary.warnings - 2} more`));
+                }
+            }
+
+            // Only show log file path if logging is enabled
+            if (this.loggingEnabled) {
+                console.log('');
+                console.log(chalk.cyan(`📄 Full log: ${chalk.white(summary.logFile)}`));
+                console.log(chalk.gray(`   View with: cat ${summary.logFile}`));
+            }
+            console.log('');
+        }
+    }
+
+    /**
+     * Finalize the log file
+     */
+    close() {
+        this.writeToFile('');
+        this.writeToFile(`Completed: ${new Date().toISOString()}`);
+        this.writeToFile(`Summary: ${this.errors.length} errors, ${this.warnings.length} warnings, ${this.infos.length} info messages`);
+        this.writeToFile('=== END LOG ===');
+    }
+
+    /**
+     * Clean up old log files (keep only last N logs)
+     */
+    static cleanupOldLogs(keepCount = 10) {
+        const logDir = '.magentrix/logs';
+
+        if (!fs.existsSync(logDir)) {
+            return;
+        }
+
+        try {
+            const files = fs.readdirSync(logDir)
+                .filter(f => f.endsWith('.log'))
+                .map(f => ({
+                    name: f,
+                    path: path.join(logDir, f),
+                    time: fs.statSync(path.join(logDir, f)).mtime.getTime()
+                }))
+                .sort((a, b) => b.time - a.time); // Sort by newest first
+
+            // Delete files beyond keepCount
+            if (files.length > keepCount) {
+                const toDelete = files.slice(keepCount);
+                toDelete.forEach(file => {
+                    try {
+                        fs.unlinkSync(file.path);
+                    } catch (err) {
+                        // Ignore errors when deleting old logs
+                    }
+                });
+            }
+        } catch (err) {
+            // Ignore errors in cleanup
+        }
+    }
+}
+
+/**
+ * Create a logger instance for an operation
+ */
+export function createLogger(operationName) {
+    return new Logger(operationName);
+}