@lightdash/cli 0.2230.0 → 0.2231.0

package/dist/handlers/download.d.ts

@@ -9,8 +9,9 @@ export type DownloadHandlerOptions = {
     skipSpaceCreate: boolean;
     public: boolean;
     includeCharts: boolean;
+    nested: boolean;
 };
-export declare const downloadContent: (ids: string[], type: "charts" | "dashboards", projectId: string, customPath?: string, languageMap?: boolean) => Promise<[number, string[]]>;
+export declare const downloadContent: (ids: string[], type: "charts" | "dashboards", projectId: string, projectName: string, customPath?: string, languageMap?: boolean, nested?: boolean) => Promise<[number, string[]]>;
 export declare const downloadHandler: (options: DownloadHandlerOptions) => Promise<void>;
 export declare const uploadHandler: (options: DownloadHandlerOptions) => Promise<void>;
 //# sourceMappingURL=download.d.ts.map

package/dist/handlers/download.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../src/handlers/download.ts"],"names":[],"mappings":"AAwBA,MAAM,MAAM,sBAAsB,GAAG;IACjC,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,KAAK,EAAE,OAAO,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,OAAO,CAAC;IACrB,eAAe,EAAE,OAAO,CAAC;IACzB,MAAM,EAAE,OAAO,CAAC;IAChB,aAAa,EAAE,OAAO,CAAC;CAC1B,CAAC;AA4IF,eAAO,MAAM,eAAe,QACnB,MAAM,EAAE,QACP,QAAQ,GAAG,YAAY,aAClB,MAAM,eACJ,MAAM,gBACN,OAAO,KACrB,OAAO,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CA2F5B,CAAC;AAEF,eAAO,MAAM,eAAe,YACf,sBAAsB,KAChC,OAAO,CAAC,IAAI,CAsHd,CAAC;AA6MF,eAAO,MAAM,aAAa,YACb,sBAAsB,KAChC,OAAO,CAAC,IAAI,CAuHd,CAAC"}
+{"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../src/handlers/download.ts"],"names":[],"mappings":"AA2BA,MAAM,MAAM,sBAAsB,GAAG;IACjC,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,KAAK,EAAE,OAAO,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,OAAO,CAAC;IACrB,eAAe,EAAE,OAAO,CAAC;IACzB,MAAM,EAAE,OAAO,CAAC;IAChB,aAAa,EAAE,OAAO,CAAC;IACvB,MAAM,EAAE,OAAO,CAAC;CACnB,CAAC;AA2NF,eAAO,MAAM,eAAe,QACnB,MAAM,EAAE,QACP,QAAQ,GAAG,YAAY,aAClB,MAAM,eACJ,MAAM,eACN,MAAM,gBACN,OAAO,WACZ,OAAO,KAChB,OAAO,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CA4F5B,CAAC;AAEF,eAAO,MAAM,eAAe,YACf,sBAAsB,KAChC,OAAO,CAAC,IAAI,CAqJd,CAAC;AA6MF,eAAO,MAAM,aAAa,YACb,sBAAsB,KAChC,OAAO,CAAC,IAAI,CAuHd,CAAC"}

package/dist/handlers/download.js

@@ -7,6 +7,7 @@ const tslib_1 = require("tslib");
 const common_1 = require("@lightdash/common");
 const fs_1 = require("fs");
 const yaml = tslib_1.__importStar(require("js-yaml"));
+const groupBy_1 = tslib_1.__importDefault(require("lodash/groupBy"));
 const path = tslib_1.__importStar(require("path"));
 const analytics_1 = require("../analytics/analytics");
 const config_1 = require("../config");
@@ -35,12 +36,19 @@ const parseContentFilters = (items) => {
     });
     return `?${new URLSearchParams(parsedItems.map((item) => ['ids', item])).toString()}`;
 };
-const createDirForContent = async (items, folder, customPath) => {
-    const outputDir = path.join(getDownloadFolder(customPath), folder);
-    globalState_1.default.debug(`Writing ${items.length} ${folder} into ${outputDir}`);
-    const created = await fs_1.promises.mkdir(outputDir, { recursive: true });
-    if (created)
-        console.info(`\nCreated new folder: ${outputDir} `);
+const createDirForContent = async (projectName, spaceSlug, folder, customPath, folderScheme) => {
+    const baseDir = getDownloadFolder(customPath);
+    let outputDir;
+    if (folderScheme === 'flat') {
+        // Flat scheme: baseDir/folder
+        outputDir = path.join(baseDir, folder);
+    }
+    else {
+        // Nested scheme: baseDir/projectName/spaceSlug/folder
+        outputDir = path.join(baseDir, projectName, spaceSlug, folder);
+    }
+    globalState_1.default.debug(`Creating directory: ${outputDir}`);
+    await fs_1.promises.mkdir(outputDir, { recursive: true });
     return outputDir;
 };
 const writeContent = async (contentAsCode, outputDir, languageMap) => {
@@ -54,56 +62,76 @@ const writeContent = async (contentAsCode, outputDir, languageMap) => {
         await fs_1.promises.writeFile(translationPath, yaml.dump(contentAsCode.translationMap));
     }
 };
+const isLightdashContentFile = (folder, entry) => entry.isFile() &&
+    entry.parentPath.endsWith(path.sep + folder) &&
+    entry.name.endsWith('.yml') &&
+    !entry.name.endsWith('.language.map.yml');
+const loadYamlFile = async (file) => {
+    const filePath = path.join(file.parentPath, file.name);
+    const [fileContent, stats] = await Promise.all([
+        fs_1.promises.readFile(filePath, 'utf-8'),
+        fs_1.promises.stat(filePath),
+    ]);
+    const item = yaml.load(fileContent);
+    const downloadedAt = item.downloadedAt
+        ? new Date(item.downloadedAt)
+        : undefined;
+    const needsUpdating = downloadedAt &&
+        Math.abs(stats.mtime.getTime() - downloadedAt.getTime()) > 30000;
+    return {
+        ...item,
+        updatedAt: needsUpdating ? stats.mtime : item.updatedAt,
+        needsUpdating: needsUpdating ?? true,
+    };
+};
 const readCodeFiles = async (folder, customPath) => {
-    const inputDir = path.join(getDownloadFolder(customPath), folder);
-    console.info(`Reading ${folder} from ${inputDir}`);
-    const items = [];
+    const baseDir = getDownloadFolder(customPath);
+    globalState_1.default.log(`Reading ${folder} from ${baseDir}`);
     try {
-        // Read all files from the lightdash directory
-        // if folder does not exist, this throws an error
-        const files = await fs_1.promises.readdir(inputDir);
-        const yamlFiles = files
-            .filter((file) => file.endsWith('.yml'))
-            .filter((file) => !file.endsWith('.language.map.yml'));
-        // Load each JSON file
-        for (const file of yamlFiles) {
-            const filePath = path.join(inputDir, file);
-            const fileContent = await fs_1.promises.readFile(filePath, 'utf-8');
-            const item = yaml.load(fileContent);
-            const fileUpdatedAt = (await fs_1.promises.stat(filePath)).mtime;
-            // We override the updatedAt to the file's updatedAt
-            // in case there were some changes made locally
-            // do not override if the file was just created
-            const downloadedAt = item.downloadedAt
-                ? new Date(item.downloadedAt)
-                : undefined;
-            const needsUpdating = downloadedAt &&
-                Math.abs(fileUpdatedAt.getTime() - downloadedAt.getTime()) >
-                    30000;
-            const locallyUpdatedItem = {
-                ...item,
-                updatedAt: needsUpdating ? fileUpdatedAt : item.updatedAt, // Force the update by changing updatedAt , which is what promotion is going to compare
-                needsUpdating: needsUpdating ?? true, // if downloadAt is not set, we force the update
-            };
-            items.push(locallyUpdatedItem);
+        const allEntries = await fs_1.promises.readdir(baseDir, {
+            recursive: true,
+            withFileTypes: true,
+        });
+        const items = await Promise.all(allEntries
+            .filter((entry) => isLightdashContentFile(folder, entry))
+            .map((file) => loadYamlFile(file)));
+        if (items.length === 0) {
+            console.error(styles.warning(`Unable to upload ${folder}, no files found in "${baseDir}". Run download command first.`));
         }
+        return items;
     }
     catch (error) {
-        // Folder does not exist
+        // Handle case where base directory doesn't exist
         if (error.code === 'ENOENT') {
-            console.error(styles.warning(`Unable to upload ${folder}, "${inputDir}" folder not found. Run download command first.`));
+            console.error(styles.warning(`Unable to upload ${folder}, "${baseDir}" folder not found. Run download command first.`));
             return [];
         }
         // Unknown error
-        console.error(styles.error(`Error reading ${inputDir}: ${error}`));
+        console.error(styles.error(`Error reading ${baseDir}: ${error}`));
         throw error;
     }
-    return items;
+};
+const groupBySpace = (items) => {
+    const itemsWithIndex = items.map((item, index) => ({ item, index }));
+    return (0, groupBy_1.default)(itemsWithIndex, (entry) => entry.item.spaceSlug);
+};
+const writeSpaceContent = async ({ projectName, spaceSlug, folder, contentInSpace, contentAsCode, customPath, languageMap, folderScheme, }) => {
+    const outputDir = await createDirForContent(projectName, spaceSlug, folder, customPath, folderScheme);
+    const contentType = folder === 'charts' ? 'chart' : 'dashboard';
+    for (const { item, index } of contentInSpace) {
+        await writeContent({
+            type: contentType,
+            content: item,
+            translationMap: contentAsCode.languageMap?.[index],
+        }, outputDir, languageMap);
+    }
 };
 const downloadContent = async (ids, // slug, uuid or url
-type, projectId, customPath, languageMap = false) => {
+type, projectId, projectName, customPath, languageMap = false, nested = false) => {
     const spinner = globalState_1.default.getActiveSpinner();
     const contentFilters = parseContentFilters(ids);
+    // Convert boolean flag to FolderScheme type
+    const folderScheme = nested ? 'nested' : 'flat';
     let contentAsCode;
     let offset = 0;
     let chartSlugs = [];
@@ -123,13 +151,20 @@ type, projectId, customPath, languageMap = false) => {
             console.warn(styles.warning(`\nNo ${type} with id "${missingId}"`));
         });
         if ('dashboards' in contentAsCode) {
-            const outputDir = await createDirForContent(contentAsCode.dashboards, 'dashboards', customPath);
-            for (const [index, dashboard,] of contentAsCode.dashboards.entries()) {
-                await writeContent({
-                    type: 'dashboard',
-                    content: dashboard,
-                    translationMap: contentAsCode.languageMap?.[index],
-                }, outputDir, languageMap);
+            // Group dashboards by spaceSlug
+            const dashboardsBySpace = groupBySpace(contentAsCode.dashboards);
+            // Create directory and write content for each space
+            for (const [spaceSlug, dashboardsInSpace] of Object.entries(dashboardsBySpace)) {
+                await writeSpaceContent({
+                    projectName,
+                    spaceSlug,
+                    folder: 'dashboards',
+                    contentInSpace: dashboardsInSpace,
+                    contentAsCode,
+                    customPath,
+                    languageMap,
+                    folderScheme,
+                });
             }
             // Extract chart slugs from dashboards
             chartSlugs = contentAsCode.dashboards.reduce((acc, dashboard) => {
@@ -143,13 +178,18 @@ type, projectId, customPath, languageMap = false) => {
             }, []);
         }
         else {
-            const outputDir = await createDirForContent(contentAsCode.charts, 'charts', customPath);
-            for (const [index, chart] of contentAsCode.charts.entries()) {
-                await writeContent({
-                    type: 'chart',
-                    content: chart,
-                    translationMap: contentAsCode.languageMap?.[index],
-                }, outputDir, languageMap);
+            const chartsBySpace = groupBySpace(contentAsCode.charts);
+            for (const [spaceSlug, chartsInSpace] of Object.entries(chartsBySpace)) {
+                await writeSpaceContent({
+                    projectName,
+                    spaceSlug,
+                    folder: 'charts',
+                    contentInSpace: chartsInSpace,
+                    contentAsCode,
+                    customPath,
+                    languageMap,
+                    folderScheme,
+                });
             }
         }
         offset = contentAsCode.offset;
@@ -159,6 +199,8 @@ type, projectId, customPath, languageMap = false) => {
 exports.downloadContent = downloadContent;
 const downloadHandler = async (options) => {
     globalState_1.default.setVerbose(options.verbose);
+    const spinner = globalState_1.default.startSpinner(`Downloading charts`);
+    spinner.start(`Downloading content from project`);
     await (0, apiClient_1.checkLightdashVersion)();
     const config = await (0, config_1.getConfig)();
     if (!config.context?.apiKey || !config.context.serverUrl) {
@@ -168,6 +210,13 @@ const downloadHandler = async (options) => {
     if (!projectId) {
         throw new Error('No project selected. Run lightdash config set-project');
     }
+    // Fetch project details to get project name for folder structure
+    const project = await (0, apiClient_1.lightdashApi)({
+        method: 'GET',
+        url: `/api/v1/projects/${projectId}`,
+        body: undefined,
+    });
+    const projectName = (0, common_1.generateSlug)(project.name);
     // For analytics
     let chartTotal;
     let dashboardTotal;
@@ -181,6 +230,16 @@ const downloadHandler = async (options) => {
         },
     });
     try {
+        const projectDir = path.join(getDownloadFolder(options.path), projectName);
+        const dirExists = () => fs_1.promises
+            .access(projectDir, fs_1.promises.constants.F_OK)
+            .then(() => true)
+            .catch(() => false);
+        // We clear the output directory first to get the latest state of content
+        // regarding projects and spaces if nested.
+        if (options.nested && (await dirExists())) {
+            await fs_1.promises.rm(projectDir, { recursive: true });
+        }
         // If any filter is provided, we skip those items without filters
         // eg: if a --charts filter is provided, we skip dashboards if no --dashboards filter is provided
         const hasFilters = options.charts.length > 0 || options.dashboards.length > 0;
@@ -189,8 +248,7 @@ const downloadHandler = async (options) => {
             console.info(styles.warning(`No charts filters provided, skipping`));
         }
         else {
-            const spinner = globalState_1.default.startSpinner(`Downloading charts`);
-            [chartTotal] = await (0, exports.downloadContent)(options.charts, 'charts', projectId, options.path, options.languageMap);
+            [chartTotal] = await (0, exports.downloadContent)(options.charts, 'charts', projectId, projectName, options.path, options.languageMap, options.nested);
             spinner.succeed(`Downloaded ${chartTotal} charts`);
         }
         // Download dashboards
@@ -198,15 +256,14 @@ const downloadHandler = async (options) => {
             console.info(styles.warning(`No dashboards filters provided, skipping`));
         }
         else {
-            const spinner = globalState_1.default.startSpinner(`Downloading dashboards`);
             let chartSlugs = [];
-            [dashboardTotal, chartSlugs] = await (0, exports.downloadContent)(options.dashboards, 'dashboards', projectId, options.path, options.languageMap);
+            [dashboardTotal, chartSlugs] = await (0, exports.downloadContent)(options.dashboards, 'dashboards', projectId, projectName, options.path, options.languageMap, options.nested);
             spinner.succeed(`Downloaded ${dashboardTotal} dashboards`);
             // If any filter is provided, we download all charts for these dashboard
             // We don't need to do this if we download everything (no filters)
             if (hasFilters && chartSlugs.length > 0) {
                 spinner.start(`Downloading ${chartSlugs.length} charts linked to dashboards`);
-                const [totalCharts] = await (0, exports.downloadContent)(chartSlugs, 'charts', projectId, options.path, options.languageMap);
+                const [totalCharts] = await (0, exports.downloadContent)(chartSlugs, 'charts', projectId, projectName, options.path, options.languageMap, options.nested);
                 spinner.succeed(`Downloaded ${totalCharts} charts linked to dashboards`);
             }
         }

package/dist/index.js

@@ -241,6 +241,7 @@ commander_1.program
     .option('-d, --dashboards <dashboards...>', 'specify dashboard slugs, uuids or urls to download', [])
     .option('-l, --language-map', 'generate a language maps for the downloaded charts and dashboards', false)
     .option('-p, --path <path>', 'specify a custom path to download charts and dashboards', undefined)
+    .option('--nested', 'organize downloads in nested folders by project and space (default: flat structure)', false)
     .option('--project <project uuid>', 'specify a project UUID to download', parseProjectArgument, undefined)
     .action(download_1.downloadHandler);
 commander_1.program

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lightdash/cli",
-  "version": "0.2230.0",
+  "version": "0.2231.0",
   "license": "MIT",
   "bin": {
     "lightdash": "dist/index.js"
@@ -34,8 +34,8 @@
     "unique-names-generator": "^4.7.1",
     "uuid": "^11.0.3",
     "yaml": "^2.7.0",
-    "@lightdash/common": "0.2230.0",
-    "@lightdash/warehouses": "0.2230.0"
+    "@lightdash/common": "0.2231.0",
+    "@lightdash/warehouses": "0.2231.0"
   },
   "description": "Lightdash CLI tool",
   "devDependencies": {