@dynamicweb/cli 1.0.14 → 1.0.16

package/README.md

@@ -86,6 +86,26 @@ Exporting all templates from current environment to local solution
 Listing the system files structure of the current environment
 > $ dw files system -lr
 
+### Files Source Type Detection
+By default, the `dw files` command automatically detects the source type based on the \<dirPath\>:
+If the path contains a file extension (e.g., 'templates/Translations.xml'), it is treated as a file.
+Otherwise, it is treated as a directory.
+In cases where this detection is incorrect, you can force the type using these flags:
+
+- `-ad` `--asDirectory`  Forces the command to treat the path as a directory, even if its name contains a dot.
+- `-af` `--asFile`       Forces the command to treat the path as a single file, even if it has no extension. 
+
+#### Examples
+
+Exporting single file from current environment to local solution
+> $ dw files templates/Translations.xml ./templates -e
+
+Exporting a directory that looks like a file
+> $ dw files templates/templates.v1 ./templates -e -ad
+
+Exporting a file that has no extension
+> $ dw files templates/testfile ./templates -e -af
+
 ### Swift
 > $ dw swift \<outPath\>
 
@@ -169,3 +189,34 @@ Config is used to manage the .dwc file through the CLI, given any prop it will c
 #### Examples
 Changing the host for the dev environment
 > $ dw config --env.dev.host localhost:6001
+
+## Using Git Bash
+If you're using Git Bash, you may encounter issues with path conversion that can interfere with relative paths used in commands.
+
+### Path Conversion Issues
+Git Bash automatically converts relative paths to absolute paths, which can cause problems. 
+You'll see a warning message if the conversion setting is not disabled:
+
+"You appear to have path conversion turned on in your shell. 
+If you are using relative paths, this may interfere. 
+Please see https://doc.dynamicweb.dev/documentation/fundamentals/code/CLI.html for more information."
+
+### Solution
+To resolve this issue, disable path conversion by setting the `MSYS_NO_PATHCONV` environment variable (current session only):
+
+> $ export MSYS_NO_PATHCONV=1
+
+#### Examples
+
+> $ export MSYS_NO_PATHCONV=1
+> $ dw files -iro ./ /TestFolder --host \<host\> --apiKey \<apiKey\>
+
+### Alternative Solutions
+If you prefer not to disable path conversion globally, you can:
+
+1. Prefix relative paths with `./` instead of just `/` to prevent conversion for specific commands.
+2. Use PowerShell or CMD instead of Git Bash.
+
+#### Examples
+
+> $ dw files -iro ./ ./TestFolder --host \<host\> --apiKey \<apiKey\>

package/bin/commands/files.js

@@ -5,7 +5,7 @@ import FormData from 'form-data';
 import { setupEnv, getAgent } from './env.js';
 import { setupUser } from './login.js';
 import { interactiveConfirm, formatBytes, createThrottledStatusUpdater } from '../utils.js';
-import { downloadWithProgress, getFileNameFromResponse } from '../downloader.js';
+import { downloadWithProgress, tryGetFileNameFromResponse } from '../downloader.js';
 import { extractWithProgress } from '../extractor.js';
 
 export function filesCommand() {
@@ -63,6 +63,18 @@ export function filesCommand() {
                 type: 'boolean',
                 describe: 'Includes export of log and cache folders, NOT RECOMMENDED'
             })
+            .option('asFile', {
+                type: 'boolean',
+                alias: 'af',
+                describe: 'Forces the command to treat the path as a single file, even if it has no extension.',
+                conflicts: 'asDirectory'
+            })
+            .option('asDirectory', {
+                type: 'boolean',
+                alias: 'ad',
+                describe: 'Forces the command to treat the path as a directory, even if its name contains a dot.',
+                conflicts: 'asFile'
+            })
         },
         handler: (argv) => {
             if (argv.verbose) console.info(`Listing directory at: ${argv.dirPath}`)
@@ -85,7 +97,19 @@ async function handleFiles(argv) {
 
     if (argv.export) {
         if (argv.dirPath) {
-            await download(env, user, argv.dirPath, argv.outPath, true, null, argv.raw, argv.iamstupid, []);
+            
+            const isFile = argv.asFile || argv.asDirectory
+                ? argv.asFile
+                : path.extname(argv.dirPath) !== '';                
+
+            if (isFile) {
+                let parentDirectory = path.dirname(argv.dirPath);              
+                parentDirectory = parentDirectory === '.' ? '/' : parentDirectory;
+                
+                await download(env, user, parentDirectory, argv.outPath, false, null, true, argv.iamstupid, [argv.dirPath], true);
+            } else {
+                await download(env, user, argv.dirPath, argv.outPath, true, null, argv.raw, argv.iamstupid, [], false);
+            }
         } else {
             await interactiveConfirm('Are you sure you want a full export of files?', async () => {
                 console.log('Full export is starting')
@@ -93,9 +117,9 @@ async function handleFiles(argv) {
                 let dirs = filesStructure.directories;
                 for (let id = 0; id < dirs.length; id++) {
                     const dir = dirs[id];
-                    await download(env, user, dir.name, argv.outPath, true, null, argv.raw, argv.iamstupid, []);
+                    await download(env, user, dir.name, argv.outPath, true, null, argv.raw, argv.iamstupid, [], false);
                 }
-                await download(env, user, '/.', argv.outPath, false, 'Base.zip', argv.raw, argv.iamstupid, Array.from(filesStructure.files.data, f => f.name));
+                await download(env, user, '/.', argv.outPath, false, 'Base.zip', argv.raw, argv.iamstupid, Array.from(filesStructure.files.data, f => f.name), false);
                 if (argv.raw) console.log('The files in the base "files" folder is in Base.zip, each directory in "files" is in its own zip')
             })
         }
@@ -165,8 +189,7 @@ function resolveTree(dirs, indentLevel, parentHasFiles) {
     }
 }
 
-async function download(env, user, dirPath, outPath, recursive, outname, raw, iamstupid, fileNames) {
-    let endpoint;
+async function download(env, user, dirPath, outPath, recursive, outname, raw, iamstupid, fileNames, singleFileMode) {
     let excludeDirectories = '';
     if (!iamstupid) {
         excludeDirectories = 'system/log';
@@ -174,19 +197,10 @@ async function download(env, user, dirPath, outPath, recursive, outname, raw, ia
             return;
         }
     }
-    let data = {
-        'DirectoryPath': dirPath ?? '/',
-        'ExcludeDirectories': [ excludeDirectories ],
-    }
 
-    if (recursive) {
-        endpoint = 'DirectoryDownload';
-    } else {
-        endpoint = 'FileDownload'
-        data['Ids'] = fileNames
-    }
+    const { endpoint, data } = prepareDownloadCommandData(dirPath, excludeDirectories, fileNames, recursive, singleFileMode);
 
-    console.log('Downloading', dirPath === '/.' ? 'Base' : dirPath, 'Recursive=' + recursive);
+    displayDownloadMessage(dirPath, fileNames, recursive, singleFileMode);
 
     const res = await fetch(`${env.protocol}://${env.host}/Admin/Api/${endpoint}`, {
         method: 'POST',
@@ -198,38 +212,81 @@ async function download(env, user, dirPath, outPath, recursive, outname, raw, ia
         agent: getAgent(env.protocol)
     });
 
-    const filename = outname || getFileNameFromResponse(res);
+    const filename = outname || tryGetFileNameFromResponse(res, dirPath);
     if (!filename) return;
 
-    let filePath = path.resolve(`${path.resolve(outPath)}/${filename}`)
-    let updater = createThrottledStatusUpdater();
+    const filePath = path.resolve(`${path.resolve(outPath)}/${filename}`)
+    const updater = createThrottledStatusUpdater();
+
     await downloadWithProgress(res, filePath, {
         onData: (received) => {
             updater.update(`Received:\t${formatBytes(received)}`);
         }
     });
+
     updater.stop();
 
-    console.log(`Finished downloading`, dirPath === '/.' ? '.' : dirPath, 'Recursive=' + recursive);
+    if (singleFileMode) {
+        console.log(`Successfully downloaded: ${filename}`);
+    } else {
+        console.log(`Finished downloading`, dirPath === '/.' ? '.' : dirPath, 'Recursive=' + recursive);
+    }
 
-    if (!raw) {
-        console.log(`\nExtracting ${filename} to ${outPath}`);
+    await extractArchive(filename, filePath, outPath, raw);
+}
 
-        const filenameWithoutExtension = filename.replace('.zip', '');
-        const destinationPath = `${path.resolve(outPath)}/${filenameWithoutExtension === 'Base' ? '' : filenameWithoutExtension}`;
+function prepareDownloadCommandData(directoryPath, excludeDirectories, fileNames, recursive, singleFileMode) {
+    const data = {
+        'DirectoryPath': directoryPath ?? '/',
+        'ExcludeDirectories': [excludeDirectories],
+    };
 
-        updater = createThrottledStatusUpdater();
-        await extractWithProgress(filePath, destinationPath, {
-            onEntry: (processedEntries, totalEntries, percent) => {
-                updater.update(`Extracted:\t${processedEntries} of ${totalEntries} files (${percent}%)`);
-            }
-        });
-        updater.stop();
+    if (recursive && !singleFileMode) {
+        return { endpoint: 'DirectoryDownload', data };
+    }
 
-        console.log(`Finished extracting ${filename} to ${outPath}\n`);
+    data['Ids'] = fileNames;
+    return { endpoint: 'FileDownload', data };
+}
 
-        fs.unlink(filePath, function(err) {});
+function displayDownloadMessage(directoryPath, fileNames, recursive, singleFileMode) {
+    if (singleFileMode) {
+        const fileName = path.basename(fileNames[0] || 'unknown');
+        console.log('Downloading file: ' + fileName);
+
+        return;
     }
+
+    const directoryPathDisplayName = directoryPath === '/.'
+        ? 'Base'
+        : directoryPath;
+
+    console.log('Downloading', directoryPathDisplayName, 'Recursive=' + recursive);
+}
+
+async function extractArchive(filename, filePath, outPath, raw) {
+    if (raw) {
+        return;
+    }
+
+    console.log(`\nExtracting ${filename} to ${outPath}`);
+    let destinationFilename = filename.replace('.zip', '');
+    if (destinationFilename === 'Base')
+        destinationFilename = '';
+
+    const destinationPath = `${path.resolve(outPath)}/${destinationFilename}`;
+    const updater = createThrottledStatusUpdater();
+
+    await extractWithProgress(filePath, destinationPath, {
+        onEntry: (processedEntries, totalEntries, percent) => {
+            updater.update(`Extracted:\t${processedEntries} of ${totalEntries} files (${percent}%)`);
+        }
+    });
+
+    updater.stop();
+    console.log(`Finished extracting ${filename} to ${outPath}\n`);
+
+    fs.unlink(filePath, function(err) {});
 }
 
 async function getFilesStructure(env, user, dirPath, recursive, includeFiles) {
@@ -251,18 +308,38 @@ async function getFilesStructure(env, user, dirPath, recursive, includeFiles) {
 
 export async function uploadFiles(env, user, localFilePaths, destinationPath, createEmpty = false, overwrite = false) {
     console.log('Uploading files')
-    let form = new FormData();
+
+    const chunkSize = 1000;
+    const chunks = [];
+
+    for (let i = 0; i < localFilePaths.length; i += chunkSize) {
+        chunks.push(localFilePaths.slice(i, i + chunkSize));
+    }
+
+    for (let i = 0; i < chunks.length; i++) {
+        console.log(`Uploading chunk ${i + 1} of ${chunks.length}`);
+
+        const chunk = chunks[i];
+        await uploadChunk(env, user, chunk, destinationPath, createEmpty, overwrite);
+
+        console.log(`Finished uploading chunk ${i + 1} of ${chunks.length}`);
+    }
+
+    console.log(`Finished uploading files. Total files: ${localFilePaths.length}, total chunks: ${chunks.length}`);
+}
+
+async function uploadChunk(env, user, filePathsChunk, destinationPath, createEmpty, overwrite) {
+    const form = new FormData();
     form.append('path', destinationPath);
     form.append('skipExistingFiles', String(!overwrite));
     form.append('allowOverwrite', String(overwrite));
-    localFilePaths.forEach((localPath, index) => {
-        let fileToUpload = resolveFilePath(localPath)
-        console.log(fileToUpload)
-        if (fileToUpload === undefined)
-            return;
+    
+    filePathsChunk.forEach(fileToUpload => {
+        console.log(`${fileToUpload}`)
         form.append('files', fs.createReadStream(path.resolve(fileToUpload)));
     });
-    let res = await fetch(`${env.protocol}://${env.host}/Admin/Api/Upload?` + new URLSearchParams({"createEmptyFiles": createEmpty, "createMissingDirectories": true}), {
+
+    const res = await fetch(`${env.protocol}://${env.host}/Admin/Api/Upload?` + new URLSearchParams({"createEmptyFiles": createEmpty, "createMissingDirectories": true}), {
         method: 'POST',
         body: form,
         headers: {
@@ -270,13 +347,13 @@ export async function uploadFiles(env, user, localFilePaths, destinationPath, cr
         },
         agent: getAgent(env.protocol)
     });
+    
     if (res.ok) {
         console.log(await res.json())
-        console.log(`Files uploaded`)
     }
     else {
         console.log(res)
-        console.log(res.json())
+        console.log(await res.json())
         process.exit(1);
     }
 }

package/bin/downloader.js

@@ -6,17 +6,35 @@ import fs from 'fs';
  * @param {Response} res - The HTTP response object.
  * @returns {string} The extracted file name.
  */
-export function getFileNameFromResponse(res) {
+export function getFileNameFromResponse(res, dirPath) {
     const header = res.headers.get('content-disposition');
     const parts = header?.split(';');
     
     if (!parts) {
-        throw new Error(`No files found in directory '${dirPath}', if you want to download all folders recursively include the -r flag`);
+        const msg = `No files found in directory '${dirPath}', if you want to download all folders recursively include the -r flag`;
+        throw new Error(msg);
     }
 
     return parts[1].split('=')[1].replace('+', ' ');
 }
 
+/**
+ * Attempts to extract the file name from an HTTP response.
+ * If extraction fails, logs the error message to the console.
+ *
+ * @param {Object} res - The HTTP response object to extract the file name from.
+ * @param {string} dirPath - The directory path to use for file name resolution.
+ * @returns {string|null} The extracted file name, or null if extraction fails.
+ */
+export function tryGetFileNameFromResponse(res, dirPath) {
+    try {
+        return getFileNameFromResponse(res, dirPath);
+    } catch (err) {
+        console.error(err.message);
+        return null;
+    }
+}
+
 /**
  * Downloads a file with progress reporting.
  * @param {Response} res - The response from which to read the stream data.

package/bin/index.js

@@ -13,6 +13,7 @@ import { queryCommand } from './commands/query.js';
 import { commandCommand } from './commands/command.js';
 
 setupConfig();
+showGitBashRelativePathWarning();
 
 yargs(hideBin(process.argv))
     .scriptName('dw')
@@ -58,4 +59,15 @@ function baseCommand() {
             console.log(`Host: ${getConfig()?.env[getConfig()?.current?.env]?.host}`)
         }
     }
-}
+}
+
+function showGitBashRelativePathWarning() {
+    const isGitBash = !!process.env.MSYSTEM;
+    const pathConversionDisabled = process.env.MSYS_NO_PATHCONV === '1';
+
+    if (isGitBash && !pathConversionDisabled) {
+        console.warn('You appear to have path conversion turned on in your shell.');
+        console.warn('If you are using relative paths, this may interfere.');
+        console.warn('Please see https://doc.dynamicweb.dev/documentation/fundamentals/code/CLI.html for more information.');
+    }
+}

package/package.json

@@ -2,7 +2,7 @@
   "name": "@dynamicweb/cli",
   "type": "module",
   "description": "Dynamicweb CLI is a commandline tool for interacting with Dynamicweb 10 solutions.",
-  "version": "1.0.14",
+  "version": "1.0.16",
   "main": "bin/index.js",
   "files": [
     "bin/*"