@lightdash/cli 0.2645.0 → 0.2646.0

package/dist/handlers/download.d.ts

@@ -14,8 +14,13 @@ export type DownloadHandlerOptions = {
     concurrency: number;
     gzip?: boolean;
 };
+type MetadataEntry = {
+    slug: string;
+    type: 'charts' | 'dashboards';
+    downloadedAt: string;
+};
 type DownloadContentType = 'charts' | 'dashboards' | 'sqlCharts';
-export declare const downloadContent: (ids: string[], type: DownloadContentType, projectId: string, projectName: string, customPath?: string, languageMap?: boolean, nested?: boolean) => Promise<[number, string[]]>;
+export declare const downloadContent: (ids: string[], type: DownloadContentType, projectId: string, projectName: string, customPath?: string, languageMap?: boolean, nested?: boolean) => Promise<[number, string[], MetadataEntry[]]>;
 export declare const downloadHandler: (options: DownloadHandlerOptions) => Promise<void>;
 export declare const uploadHandler: (options: DownloadHandlerOptions) => Promise<void>;
 export {};

package/dist/handlers/download.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../src/handlers/download.ts"],"names":[],"mappings":"AAqCA,MAAM,MAAM,sBAAsB,GAAG;IACjC,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,KAAK,EAAE,OAAO,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,OAAO,CAAC;IACrB,eAAe,EAAE,OAAO,CAAC;IACzB,MAAM,EAAE,OAAO,CAAC;IAChB,aAAa,EAAE,OAAO,CAAC;IACvB,MAAM,EAAE,OAAO,CAAC;IAChB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,CAAC,EAAE,OAAO,CAAC;CAClB,CAAC;AAmQF,KAAK,mBAAmB,GAAG,QAAQ,GAAG,YAAY,GAAG,WAAW,CAAC;AAkDjE,eAAO,MAAM,eAAe,GACxB,KAAK,MAAM,EAAE,EACb,MAAM,mBAAmB,EACzB,WAAW,MAAM,EACjB,aAAa,MAAM,EACnB,aAAa,MAAM,EACnB,cAAa,OAAe,EAC5B,SAAQ,OAAe,KACxB,OAAO,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CA2G5B,CAAC;AAEF,eAAO,MAAM,eAAe,GACxB,SAAS,sBAAsB,KAChC,OAAO,CAAC,IAAI,CA0Kd,CAAC;AA4ZF,eAAO,MAAM,aAAa,GACtB,SAAS,sBAAsB,KAChC,OAAO,CAAC,IAAI,CAkJd,CAAC"}
+{"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../src/handlers/download.ts"],"names":[],"mappings":"AA2CA,MAAM,MAAM,sBAAsB,GAAG;IACjC,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,KAAK,EAAE,OAAO,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,OAAO,CAAC;IACrB,eAAe,EAAE,OAAO,CAAC;IACzB,MAAM,EAAE,OAAO,CAAC;IAChB,aAAa,EAAE,OAAO,CAAC;IACvB,MAAM,EAAE,OAAO,CAAC;IAChB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,CAAC,EAAE,OAAO,CAAC;CAClB,CAAC;AA2FF,KAAK,aAAa,GAAG;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,QAAQ,GAAG,YAAY,CAAC;IAC9B,YAAY,EAAE,MAAM,CAAC;CACxB,CAAC;AAyOF,KAAK,mBAAmB,GAAG,QAAQ,GAAG,YAAY,GAAG,WAAW,CAAC;AAkDjE,eAAO,MAAM,eAAe,GACxB,KAAK,MAAM,EAAE,EACb,MAAM,mBAAmB,EACzB,WAAW,MAAM,EACjB,aAAa,MAAM,EACnB,aAAa,MAAM,EACnB,cAAa,OAAe,EAC5B,SAAQ,OAAe,KACxB,OAAO,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,aAAa,EAAE,CAAC,CA+G7C,CAAC;AAEF,eAAO,MAAM,eAAe,GACxB,SAAS,sBAAsB,KAChC,OAAO,CAAC,IAAI,CAmMd,CAAC;AA4ZF,eAAO,MAAM,aAAa,GACtB,SAAS,sBAAsB,KAChC,OAAO,CAAC,IAAI,CAkJd,CAAC"}

package/dist/handlers/download.js

@@ -15,6 +15,7 @@ const config_1 = require("../config");
 const globalState_1 = tslib_1.__importDefault(require("../globalState"));
 const styles = tslib_1.__importStar(require("../styles"));
 const apiClient_1 = require("./dbt/apiClient");
+const metadataFile_1 = require("./metadataFile");
 const selectProject_1 = require("./selectProject");
 const getDownloadFolder = (customPath) => {
     if (customPath) {
@@ -71,29 +72,69 @@ const getFileExtension = (contentType) => {
 const writeContent = async (contentAsCode, outputDir, languageMap) => {
     const extension = getFileExtension(contentAsCode.type);
     const itemPath = path.join(outputDir, `${contentAsCode.content.slug}${extension}`);
-    const chartYml = yaml.dump(contentAsCode.content, {
+    // Strip timestamps — they go to .lightdash-metadata.json instead
+    const { updatedAt, downloadedAt, ...cleanContent } = contentAsCode.content;
+    const chartYml = yaml.dump(cleanContent, {
         quotingType: '"',
+        sortKeys: true,
     });
     await fs_1.promises.writeFile(itemPath, chartYml);
     if (contentAsCode.translationMap && languageMap) {
         const translationPath = path.join(outputDir, `${contentAsCode.content.slug}.language.map.yml`);
-        await fs_1.promises.writeFile(translationPath, yaml.dump(contentAsCode.translationMap));
+        await fs_1.promises.writeFile(translationPath, yaml.dump(contentAsCode.translationMap, { sortKeys: true }));
     }
+    const metadataType = contentAsCode.type === 'dashboard' ? 'dashboards' : 'charts';
+    let downloadedAtString;
+    if (downloadedAt instanceof Date) {
+        downloadedAtString = downloadedAt.toISOString();
+    }
+    else if (typeof downloadedAt === 'string') {
+        downloadedAtString = downloadedAt;
+    }
+    else {
+        downloadedAtString = new Date().toISOString();
+    }
+    return {
+        slug: contentAsCode.content.slug,
+        type: metadataType,
+        downloadedAt: downloadedAtString,
+    };
+};
+const hasUnsortedKeys = (obj) => {
+    if (typeof obj !== 'object' || obj === null || Array.isArray(obj)) {
+        if (Array.isArray(obj)) {
+            return obj.some(hasUnsortedKeys);
+        }
+        return false;
+    }
+    const keys = Object.keys(obj);
+    const sorted = [...keys].sort();
+    if (keys.some((key, i) => key !== sorted[i])) {
+        return true;
+    }
+    return Object.values(obj).some(hasUnsortedKeys);
 };
 const isLightdashContentFile = (folder, entry) => entry.isFile() &&
     entry.parentPath &&
     entry.parentPath.endsWith(path.sep + folder) &&
     entry.name.endsWith('.yml') &&
     !entry.name.endsWith('.language.map.yml');
-const loadYamlFile = async (file) => {
+const loadYamlFile = async (file, folder, metadata) => {
     const filePath = path.join(file.parentPath, file.name);
     const [fileContent, stats] = await Promise.all([
         fs_1.promises.readFile(filePath, 'utf-8'),
         fs_1.promises.stat(filePath),
     ]);
     const item = yaml.load(fileContent);
-    const downloadedAt = item.downloadedAt
-        ? new Date(item.downloadedAt)
+    if (hasUnsortedKeys(item)) {
+        globalState_1.default.log(styles.warning(`Warning: ${file.name} has unsorted YAML keys. Re-download to fix, or sort keys alphabetically.`));
+    }
+    const metadataSection = folder === 'dashboards' ? metadata.dashboards : metadata.charts;
+    const downloadedAtRaw = metadataSection[item.slug] ?? item.downloadedAt;
+    const downloadedAt = downloadedAtRaw
+        ? new Date(downloadedAtRaw instanceof Date
+            ? downloadedAtRaw.getTime()
+            : downloadedAtRaw)
         : undefined;
     const needsUpdating = downloadedAt &&
         Math.abs(stats.mtime.getTime() - downloadedAt.getTime()) > 30000;
@@ -111,13 +152,14 @@ const readCodeFiles = async (folder, customPath) => {
         throw new Error(`Node.js v20.12.0 or later is required for this command (current: ${process.version}).`);
     }
     try {
+        const metadata = await (0, metadataFile_1.readMetadataFile)(baseDir);
         const allEntries = await fs_1.promises.readdir(baseDir, {
             recursive: true,
             withFileTypes: true,
         });
         const items = await Promise.all(allEntries
             .filter((entry) => isLightdashContentFile(folder, entry))
-            .map((file) => loadYamlFile(file)));
+            .map((file) => loadYamlFile(file, folder, metadata)));
         if (items.length === 0) {
             console.error(styles.warning(`Unable to upload ${folder}, no files found in "${baseDir}". Run download command first.`));
         }
@@ -140,16 +182,19 @@ const groupBySpace = (items) => {
 };
 const writeSpaceContent = async ({ projectName, spaceSlug, folder, contentType, contentInSpace, contentAsCode, customPath, languageMap, folderScheme, }) => {
     const outputDir = await createDirForContent(projectName, spaceSlug, folder, customPath, folderScheme);
+    const entries = [];
     for (const { item, index } of contentInSpace) {
         const translationMap = 'languageMap' in contentAsCode
             ? contentAsCode.languageMap?.[index]
             : undefined;
-        await writeContent({
+        const entry = await writeContent({
             type: contentType,
             content: item,
             translationMap,
         }, outputDir, languageMap);
+        entries.push(entry);
     }
+    return entries;
 };
 const getContentTypeConfig = (type, projectId) => {
     switch (type) {
@@ -191,6 +236,7 @@ const downloadContent = async (ids, type, projectId, projectName, customPath, la
     let offset = 0;
     let total = 0;
     let chartSlugs = [];
+    let allMetadataEntries = [];
     do {
         globalState_1.default.debug(`Downloading ${config.displayName} with offset "${offset}" and filters "${contentFilters}"`);
         const commonParams = config.supportsLanguageMap
@@ -214,7 +260,7 @@ const downloadContent = async (ids, type, projectId, projectName, customPath, la
         if ('sqlCharts' in results) {
             const sqlChartsBySpace = groupBySpace(results.sqlCharts);
             for (const [spaceSlug, sqlChartsInSpace] of Object.entries(sqlChartsBySpace)) {
-                await writeSpaceContent({
+                const entries = await writeSpaceContent({
                     projectName,
                     spaceSlug,
                     folder: 'charts',
@@ -225,12 +271,13 @@ const downloadContent = async (ids, type, projectId, projectName, customPath, la
                     languageMap,
                     folderScheme,
                 });
+                allMetadataEntries = [...allMetadataEntries, ...entries];
             }
         }
         else if ('dashboards' in results) {
             const dashboardsBySpace = groupBySpace(results.dashboards);
             for (const [spaceSlug, dashboardsInSpace] of Object.entries(dashboardsBySpace)) {
-                await writeSpaceContent({
+                const entries = await writeSpaceContent({
                     projectName,
                     spaceSlug,
                     folder: 'dashboards',
@@ -241,6 +288,7 @@ const downloadContent = async (ids, type, projectId, projectName, customPath, la
                     languageMap,
                     folderScheme,
                 });
+                allMetadataEntries = [...allMetadataEntries, ...entries];
             }
             chartSlugs = [
                 ...chartSlugs,
@@ -250,7 +298,7 @@ const downloadContent = async (ids, type, projectId, projectName, customPath, la
         else {
             const chartsBySpace = groupBySpace(results.charts);
             for (const [spaceSlug, chartsInSpace] of Object.entries(chartsBySpace)) {
-                await writeSpaceContent({
+                const entries = await writeSpaceContent({
                     projectName,
                     spaceSlug,
                     folder: 'charts',
@@ -261,12 +309,13 @@ const downloadContent = async (ids, type, projectId, projectName, customPath, la
                     languageMap,
                     folderScheme,
                 });
+                allMetadataEntries = [...allMetadataEntries, ...entries];
             }
         }
         offset = results.offset;
         total = results.total;
     } while (offset < total);
-    return [total, [...new Set(chartSlugs)]];
+    return [total, [...new Set(chartSlugs)], allMetadataEntries];
 };
 exports.downloadContent = downloadContent;
 const downloadHandler = async (options) => {
@@ -310,20 +359,21 @@ const downloadHandler = async (options) => {
         },
     });
     try {
-        // If any filter is provided, we skip those items without filters
-        // eg: if a --charts filter is provided, we skip dashboards if no --dashboards filter is provided
         const hasFilters = options.charts.length > 0 || options.dashboards.length > 0;
+        let allMetadataEntries = [];
         // Download regular charts
         if (hasFilters && options.charts.length === 0) {
             console.info(styles.warning(`No charts filters provided, skipping`));
         }
         else {
-            const [regularChartTotal] = await (0, exports.downloadContent)(options.charts, 'charts', projectId, projectName, options.path, options.languageMap, options.nested);
+            const [regularChartTotal, , regularChartMeta] = await (0, exports.downloadContent)(options.charts, 'charts', projectId, projectName, options.path, options.languageMap, options.nested);
             spinner.succeed(`Downloaded ${regularChartTotal} charts`);
+            allMetadataEntries = [...allMetadataEntries, ...regularChartMeta];
             // Download SQL charts
             spinner.start(`Downloading SQL charts`);
-            const [sqlChartTotal] = await (0, exports.downloadContent)(options.charts, 'sqlCharts', projectId, projectName, options.path, options.languageMap, options.nested);
+            const [sqlChartTotal, , sqlChartMeta] = await (0, exports.downloadContent)(options.charts, 'sqlCharts', projectId, projectName, options.path, options.languageMap, options.nested);
             spinner.succeed(`Downloaded ${sqlChartTotal} SQL charts`);
+            allMetadataEntries = [...allMetadataEntries, ...sqlChartMeta];
             chartTotal = regularChartTotal + sqlChartTotal;
         }
         // Download dashboards
@@ -332,18 +382,34 @@ const downloadHandler = async (options) => {
         }
         else {
             let chartSlugs = [];
-            [dashboardTotal, chartSlugs] = await (0, exports.downloadContent)(options.dashboards, 'dashboards', projectId, projectName, options.path, options.languageMap, options.nested);
+            let dashMeta;
+            [dashboardTotal, chartSlugs, dashMeta] = await (0, exports.downloadContent)(options.dashboards, 'dashboards', projectId, projectName, options.path, options.languageMap, options.nested);
+            allMetadataEntries = [...allMetadataEntries, ...dashMeta];
             spinner.succeed(`Downloaded ${dashboardTotal} dashboards`);
-            // If any filter is provided, we download all charts linked to these dashboards
-            // We don't need to do this if we download everything (no filters)
             if (hasFilters && chartSlugs.length > 0) {
                 spinner.start(`Downloading ${chartSlugs.length} charts linked to dashboards`);
-                // Download both regular charts and SQL charts linked to dashboards
-                const [regularCharts] = await (0, exports.downloadContent)(chartSlugs, 'charts', projectId, projectName, options.path, options.languageMap, options.nested);
-                const [sqlCharts] = await (0, exports.downloadContent)(chartSlugs, 'sqlCharts', projectId, projectName, options.path, options.languageMap, options.nested);
+                const [regularCharts, , linkedChartMeta] = await (0, exports.downloadContent)(chartSlugs, 'charts', projectId, projectName, options.path, options.languageMap, options.nested);
+                allMetadataEntries = [
+                    ...allMetadataEntries,
+                    ...linkedChartMeta,
+                ];
+                const [sqlCharts, , linkedSqlMeta] = await (0, exports.downloadContent)(chartSlugs, 'sqlCharts', projectId, projectName, options.path, options.languageMap, options.nested);
+                allMetadataEntries = [...allMetadataEntries, ...linkedSqlMeta];
                 spinner.succeed(`Downloaded ${regularCharts + sqlCharts} charts linked to dashboards`);
             }
         }
+        // Write metadata file with all downloadedAt timestamps
+        const metadataToWrite = {
+            version: 1,
+            charts: {},
+            dashboards: {},
+        };
+        for (const entry of allMetadataEntries) {
+            metadataToWrite[entry.type][entry.slug] = entry.downloadedAt;
+        }
+        const baseDir = getDownloadFolder(options.path);
+        await (0, metadataFile_1.writeMetadataFile)(baseDir, metadataToWrite);
+        globalState_1.default.log(styles.warning(`\nNote: ${metadataFile_1.METADATA_FILENAME} was written to ${baseDir}. Consider adding it to your .gitignore.`));
         const end = Date.now();
         await analytics_1.LightdashAnalytics.track({
             event: 'download.completed',
@@ -353,7 +419,7 @@ const downloadHandler = async (options) => {
                 projectId,
                 chartsNum: chartTotal,
                 dashboardsNum: dashboardTotal,
-                timeToCompleted: (end - start) / 1000, // in seconds
+                timeToCompleted: (end - start) / 1000,
             },
         });
     }

package/dist/handlers/metadataFile.d.ts

@@ -0,0 +1,9 @@
+export declare const METADATA_FILENAME = ".lightdash-metadata.json";
+export type LightdashMetadata = {
+    version: 1;
+    charts: Record<string, string>;
+    dashboards: Record<string, string>;
+};
+export declare const readMetadataFile: (baseDir: string) => Promise<LightdashMetadata>;
+export declare const writeMetadataFile: (baseDir: string, metadata: LightdashMetadata) => Promise<void>;
+//# sourceMappingURL=metadataFile.d.ts.map

package/dist/handlers/metadataFile.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"metadataFile.d.ts","sourceRoot":"","sources":["../../src/handlers/metadataFile.ts"],"names":[],"mappings":"AAGA,eAAO,MAAM,iBAAiB,6BAA6B,CAAC;AAE5D,MAAM,MAAM,iBAAiB,GAAG;IAC5B,OAAO,EAAE,CAAC,CAAC;IACX,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC/B,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACtC,CAAC;AAQF,eAAO,MAAM,gBAAgB,GACzB,SAAS,MAAM,KAChB,OAAO,CAAC,iBAAiB,CAQ3B,CAAC;AAEF,eAAO,MAAM,iBAAiB,GAC1B,SAAS,MAAM,EACf,UAAU,iBAAiB,KAC5B,OAAO,CAAC,IAAI,CASd,CAAC"}

package/dist/handlers/metadataFile.js

@@ -0,0 +1,34 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.writeMetadataFile = exports.readMetadataFile = exports.METADATA_FILENAME = void 0;
+const tslib_1 = require("tslib");
+const fs_1 = require("fs");
+const path = tslib_1.__importStar(require("path"));
+exports.METADATA_FILENAME = '.lightdash-metadata.json';
+const emptyMetadata = () => ({
+    version: 1,
+    charts: {},
+    dashboards: {},
+});
+const readMetadataFile = async (baseDir) => {
+    const filePath = path.join(baseDir, exports.METADATA_FILENAME);
+    try {
+        const content = await fs_1.promises.readFile(filePath, 'utf-8');
+        return JSON.parse(content);
+    }
+    catch {
+        return emptyMetadata();
+    }
+};
+exports.readMetadataFile = readMetadataFile;
+const writeMetadataFile = async (baseDir, metadata) => {
+    const existing = await (0, exports.readMetadataFile)(baseDir);
+    const merged = {
+        version: 1,
+        charts: { ...existing.charts, ...metadata.charts },
+        dashboards: { ...existing.dashboards, ...metadata.dashboards },
+    };
+    const filePath = path.join(baseDir, exports.METADATA_FILENAME);
+    await fs_1.promises.writeFile(filePath, JSON.stringify(merged, null, 2));
+};
+exports.writeMetadataFile = writeMetadataFile;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lightdash/cli",
-  "version": "0.2645.0",
+  "version": "0.2646.0",
   "description": "Lightdash CLI tool",
   "license": "MIT",
   "repository": {
@@ -40,8 +40,8 @@
     "unique-names-generator": "^4.7.1",
     "uuid": "^11.0.3",
     "yaml": "^2.7.0",
-    "@lightdash/common": "0.2645.0",
-    "@lightdash/warehouses": "0.2645.0"
+    "@lightdash/common": "0.2646.0",
+    "@lightdash/warehouses": "0.2646.0"
   },
   "devDependencies": {
     "@types/inquirer": "^8.2.1",