@lightdash/cli 0.1391.0 → 0.1393.0

package/dist/handlers/download.d.ts

@@ -1,5 +1,6 @@
 export type DownloadHandlerOptions = {
     verbose: boolean;
+    force: boolean;
 };
 export declare const downloadHandler: (options: DownloadHandlerOptions) => Promise<void>;
 export declare const uploadHandler: (options: DownloadHandlerOptions) => Promise<void>;

package/dist/handlers/download.js

@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.uploadHandler = exports.downloadHandler = void 0;
 const tslib_1 = require("tslib");
 /* eslint-disable no-await-in-loop */
+/* eslint-disable no-param-reassign */
 const common_1 = require("@lightdash/common");
 const fs_1 = require("fs");
 const yaml = tslib_1.__importStar(require("js-yaml"));
@@ -11,56 +12,25 @@ const config_1 = require("../config");
 const globalState_1 = tslib_1.__importDefault(require("../globalState"));
 const apiClient_1 = require("./dbt/apiClient");
 const DOWNLOAD_FOLDER = 'lightdash';
-const downloadHandler = async (options) => {
-    globalState_1.default.setVerbose(options.verbose);
-    await (0, apiClient_1.checkLightdashVersion)();
-    const config = await (0, config_1.getConfig)();
-    if (!config.context?.apiKey || !config.context.serverUrl) {
-        throw new common_1.AuthorizationError(`Not logged in. Run 'lightdash login --help'`);
-    }
-    const projectId = config.context.project;
-    if (!projectId) {
-        throw new Error('No project selected. Run lightdash config set-project');
-    }
-    const chartsAsCode = await (0, apiClient_1.lightdashApi)({
-        method: 'GET',
-        url: `/api/v1/projects/${projectId}/charts/code`,
-        body: undefined,
-    });
-    console.info(`Downloading ${chartsAsCode.length} charts`);
-    const outputDir = path.join(process.cwd(), DOWNLOAD_FOLDER);
-    console.info(`Creating new path for files on ${outputDir} `);
-    try {
-        await fs_1.promises.mkdir(outputDir, { recursive: true });
-    }
-    catch (error) {
-        // Directory already exists
-    }
-    for (const chart of chartsAsCode) {
-        const chartPath = path.join(outputDir, `${chart.slug}.yml`);
-        globalState_1.default.debug(`> Writing chart to ${chartPath}`);
-        const chartYml = yaml.dump(chart, {
+const dumpIntoFiles = async (folder, items) => {
+    const outputDir = path.join(process.cwd(), DOWNLOAD_FOLDER, folder);
+    console.info(`Writting ${items.length} ${folder} into ${outputDir}`);
+    // Make directory
+    const created = await fs_1.promises.mkdir(outputDir, { recursive: true });
+    if (created)
+        console.info(`Created new folder: ${outputDir} `);
+    for (const item of items) {
+        const itemPath = path.join(outputDir, `${item.slug}.yml`);
+        const chartYml = yaml.dump(item, {
             quotingType: '"',
         });
-        await fs_1.promises.writeFile(chartPath, chartYml);
+        await fs_1.promises.writeFile(itemPath, chartYml);
     }
-    // TODO delete files if chart don't exist ?*/
 };
-exports.downloadHandler = downloadHandler;
-const uploadHandler = async (options) => {
-    globalState_1.default.setVerbose(options.verbose);
-    await (0, apiClient_1.checkLightdashVersion)();
-    const config = await (0, config_1.getConfig)();
-    if (!config.context?.apiKey || !config.context.serverUrl) {
-        throw new common_1.AuthorizationError(`Not logged in. Run 'lightdash login --help'`);
-    }
-    const projectId = config.context.project;
-    if (!projectId) {
-        throw new Error('No project selected. Run lightdash config set-project');
-    }
-    const inputDir = path.join(process.cwd(), DOWNLOAD_FOLDER);
-    console.info(`Reading charts from ${inputDir}`);
-    const charts = [];
+const readCodeFiles = async (folder) => {
+    const inputDir = path.join(process.cwd(), DOWNLOAD_FOLDER, folder);
+    console.info(`Reading ${folder} from ${inputDir}`);
+    const items = [];
     try {
         // Read all files from the lightdash directory
         const files = await fs_1.promises.readdir(inputDir);
@@ -69,91 +39,138 @@ const uploadHandler = async (options) => {
         for (const file of jsonFiles) {
             const filePath = path.join(inputDir, file);
             const fileContent = await fs_1.promises.readFile(filePath, 'utf-8');
-            const chart = yaml.load(fileContent);
+            const item = yaml.load(fileContent);
             const fileUpdatedAt = (await fs_1.promises.stat(filePath)).mtime;
             // We override the updatedAt to the file's updatedAt
             // in case there were some changes made locally
             // do not override if the file was just created
-            const downloadedAt = chart.downloadedAt
-                ? new Date(chart.downloadedAt)
+            const downloadedAt = item.downloadedAt
+                ? new Date(item.downloadedAt)
                 : undefined;
             const needsUpdating = downloadedAt &&
                 Math.abs(fileUpdatedAt.getTime() - downloadedAt.getTime()) >
                     30000;
-            const locallyUpdatedChart = {
-                ...chart,
-                updatedAt: needsUpdating ? fileUpdatedAt : chart.updatedAt,
+            const locallyUpdatedItem = {
+                ...item,
+                updatedAt: needsUpdating ? fileUpdatedAt : item.updatedAt,
                 needsUpdating: needsUpdating ?? true, // if downloadAt is not set, we force the update
             };
-            charts.push(locallyUpdatedChart);
+            items.push(locallyUpdatedItem);
         }
     }
     catch (error) {
         if (error.code === 'ENOENT') {
-            throw new Error(`Directory ${inputDir} not found. Run download command first.`);
+            console.error(`Directory ${inputDir} not found. Run download command first.`);
+        }
+        else {
+            console.error(`Error reading ${inputDir}: ${error}`);
         }
         throw error;
     }
-    console.info(`Found ${charts.length} chart files`);
-    let created = 0;
-    let updated = 0;
-    let deleted = 0;
-    let skipped = 0;
-    let spacesCreated = 0;
-    let spacesUpdated = 0;
-    try {
-        for (const chart of charts) {
-            if (!chart.needsUpdating) {
-                globalState_1.default.debug(`Skipping chart "${chart.slug}" with no local changes`);
-                skipped += 1;
+    return items;
+};
+const downloadHandler = async (options) => {
+    globalState_1.default.setVerbose(options.verbose);
+    await (0, apiClient_1.checkLightdashVersion)();
+    const config = await (0, config_1.getConfig)();
+    if (!config.context?.apiKey || !config.context.serverUrl) {
+        throw new common_1.AuthorizationError(`Not logged in. Run 'lightdash login --help'`);
+    }
+    const projectId = config.context.project;
+    if (!projectId) {
+        throw new Error('No project selected. Run lightdash config set-project');
+    }
+    // Download charts
+    globalState_1.default.debug('Downloading charts');
+    const chartsAsCode = await (0, apiClient_1.lightdashApi)({
+        method: 'GET',
+        url: `/api/v1/projects/${projectId}/charts/code`,
+        body: undefined,
+    });
+    await dumpIntoFiles('charts', chartsAsCode);
+    // Download dashboards
+    globalState_1.default.debug('Downloading dashboards');
+    const dashboardsAsCode = await (0, apiClient_1.lightdashApi)({
+        method: 'GET',
+        url: `/api/v1/projects/${projectId}/dashboards/code`,
+        body: undefined,
+    });
+    await dumpIntoFiles('dashboards', dashboardsAsCode);
+    // TODO delete files if chart don't exist ?*/
+};
+exports.downloadHandler = downloadHandler;
+const storeUploadChanges = (changes, promoteChanges) => {
+    const getPromoteChanges = (resource) => {
+        const promotions = promoteChanges[resource];
+        return promotions.reduce((acc, promoteChange) => {
+            const action = promoteChange.action === common_1.PromotionAction.NO_CHANGES
+                ? 'skipped'
+                : promoteChange.action;
+            const key = `${resource} ${action}`;
+            acc[key] = (acc[key] ?? 0) + 1;
+            return acc;
+        }, {});
+    };
+    const updatedChanges = {
+        ...changes,
+    };
+    ['spaces', 'charts', 'dashboards'].forEach((resource) => {
+        const resourceChanges = getPromoteChanges(resource);
+        Object.entries(resourceChanges).forEach(([key, value]) => {
+            updatedChanges[key] = (updatedChanges[key] ?? 0) + value;
+        });
+    });
+    return updatedChanges;
+};
+const logUploadChanges = (changes) => {
+    Object.entries(changes).forEach(([key, value]) => {
+        console.info(`Total ${key}: ${value} `);
+    });
+};
+const upsertResources = async (type, projectId, changes, force) => {
+    const items = await readCodeFiles(type);
+    console.info(`Found ${items.length} ${type} files`);
+    for (const item of items) {
+        // If a chart fails to update, we keep updating the rest
+        try {
+            if (!force && !item.needsUpdating) {
+                globalState_1.default.debug(`Skipping ${type} "${item.slug}" with no local changes`);
+                changes[`${type} skipped`] =
+                    (changes[`${type} skipped`] ?? 0) + 1;
                 // eslint-disable-next-line no-continue
                 continue;
             }
-            globalState_1.default.debug(`Upserting chart ${chart.slug}`);
-            const chartData = await (0, apiClient_1.lightdashApi)({
+            globalState_1.default.debug(`Upserting ${type} ${item.slug}`);
+            const upsertData = await (0, apiClient_1.lightdashApi)({
                 method: 'POST',
-                url: `/api/v1/projects/${projectId}/charts/${chart.slug}/code`,
-                body: JSON.stringify(chart),
+                url: `/api/v1/projects/${projectId}/${type}/${item.slug}/code`,
+                body: JSON.stringify(item),
             });
-            globalState_1.default.debug(`Chart "${chart.name}": ${chartData.charts[0].action}`);
-            switch (chartData.spaces[0].action) {
-                case common_1.PromotionAction.CREATE:
-                    spacesCreated += 1;
-                    break;
-                case common_1.PromotionAction.UPDATE:
-                    spacesUpdated += 1;
-                    break;
-                default:
-                // ignore the rest
-            }
-            switch (chartData.charts[0].action) {
-                case common_1.PromotionAction.CREATE:
-                    created += 1;
-                    break;
-                case common_1.PromotionAction.UPDATE:
-                    updated += 1;
-                    break;
-                case common_1.PromotionAction.DELETE:
-                    deleted += 1;
-                    break;
-                case common_1.PromotionAction.NO_CHANGES:
-                    skipped += 1;
-                    break;
-                default:
-                    globalState_1.default.debug(`Unknown action: ${chartData.charts[0].action}`);
-                    break;
-            }
+            globalState_1.default.debug(`${type} "${item.name}": ${upsertData[type]?.[0].action}`);
+            changes = storeUploadChanges(changes, upsertData);
+        }
+        catch (error) {
+            changes[`${type} with errors`] =
+                (changes[`${type} with errors`] ?? 0) + 1;
+            console.error(`Error upserting ${type}`, error);
         }
     }
-    catch (error) {
-        console.error('Error upserting chart', error);
+    return changes;
+};
+const uploadHandler = async (options) => {
+    globalState_1.default.setVerbose(options.verbose);
+    await (0, apiClient_1.checkLightdashVersion)();
+    const config = await (0, config_1.getConfig)();
+    if (!config.context?.apiKey || !config.context.serverUrl) {
+        throw new common_1.AuthorizationError(`Not logged in. Run 'lightdash login --help'`);
+    }
+    const projectId = config.context.project;
+    if (!projectId) {
+        throw new Error('No project selected. Run lightdash config set-project');
     }
-    console.info(`Total charts created: ${created} `);
-    console.info(`Total charts updated: ${updated} `);
-    console.info(`Total charts skipped: ${skipped} `);
-    if (deleted > 0)
-        console.info(`Total charts deleted: ${deleted} `); // We should not delete charts from the CLI
-    console.info(`Total spaces created: ${spacesCreated} `);
-    console.info(`Total spaces updated: ${spacesUpdated} `);
+    let changes = {};
+    changes = await upsertResources('charts', projectId, changes, options.force);
+    changes = await upsertResources('dashboards', projectId, changes, options.force);
+    logUploadChanges(changes);
 };
 exports.uploadHandler = uploadHandler;

package/dist/index.js

@@ -199,6 +199,7 @@ commander_1.program
     .command('upload')
     .description('Uploads charts and dashboards as code')
     .option('--verbose', undefined, false)
+    .option('--force', 'Force upload even if local files have not changed, use this when you want to upload files to a new project', false)
     .action(download_1.uploadHandler);
 commander_1.program
     .command('deploy')

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@lightdash/cli",
-    "version": "0.1391.0",
+    "version": "0.1393.0",
     "license": "MIT",
     "bin": {
         "lightdash": "dist/index.js"
@@ -11,8 +11,8 @@
     ],
     "dependencies": {
         "@actions/core": "^1.11.1",
-        "@lightdash/common": "^0.1391.0",
-        "@lightdash/warehouses": "^0.1391.0",
+        "@lightdash/common": "^0.1393.0",
+        "@lightdash/warehouses": "^0.1393.0",
         "@types/columnify": "^1.5.1",
         "ajv": "^8.11.0",
         "ajv-formats": "^2.1.1",