@claryai/cli 0.1.0

package/dist/handlers/download.js

@@ -0,0 +1,955 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.uploadHandler = exports.downloadHandler = exports.downloadContent = void 0;
+const tslib_1 = require("tslib");
+/* eslint-disable no-await-in-loop */
+/* eslint-disable no-param-reassign */
+const common_1 = require("@lightdash/common");
+const fs_1 = require("fs");
+const yaml = tslib_1.__importStar(require("js-yaml"));
+const groupBy_1 = tslib_1.__importDefault(require("lodash/groupBy"));
+const p_limit_1 = tslib_1.__importDefault(require("p-limit"));
+const path = tslib_1.__importStar(require("path"));
+const analytics_1 = require("../analytics/analytics");
+const config_1 = require("../config");
+const globalState_1 = tslib_1.__importDefault(require("../globalState"));
+const styles = tslib_1.__importStar(require("../styles"));
+const apiClient_1 = require("./dbt/apiClient");
+const metadataFile_1 = require("./metadataFile");
+const selectProject_1 = require("./selectProject");
+const getDownloadFolder = (customPath) => {
+    if (customPath) {
+        return path.isAbsolute(customPath)
+            ? customPath
+            : path.join(process.cwd(), customPath);
+    }
+    return path.join(process.cwd(), 'clary');
+};
+/*
+    This function is used to parse the content filters.
+    It can be slugs, uuids or urls
+    We remove the URL part (if any) and return a list of `slugs or uuids` that can be used in the API call
+*/
+const parseContentFilters = (items) => {
+    if (items.length === 0)
+        return '';
+    const parsedItems = items.map((item) => {
+        const uuidMatch = item.match(/https?:\/\/.+\/(?:saved|dashboards)\/([a-f0-9-]+)/i);
+        return uuidMatch ? uuidMatch[1] : item;
+    });
+    return `?${new URLSearchParams(parsedItems.map((item) => ['ids', item])).toString()}`;
+};
+const createDirForContent = async (projectName, spaceSlug, folder, customPath, folderScheme) => {
+    const baseDir = getDownloadFolder(customPath);
+    let outputDir;
+    if (folderScheme === 'flat') {
+        // Flat scheme: baseDir/folder
+        outputDir = path.join(baseDir, folder);
+    }
+    else {
+        // Nested scheme: baseDir/projectName/spaceSlug/folder
+        outputDir = path.join(baseDir, projectName, spaceSlug, folder);
+    }
+    globalState_1.default.debug(`Creating directory: ${outputDir}`);
+    await fs_1.promises.mkdir(outputDir, { recursive: true });
+    return outputDir;
+};
+/**
+ * Get file extension for content-as-code files.
+ * SQL charts use '.sql.yml' extension to avoid filename conflicts with regular charts
+ * that may have the same slug, since both chart types share the same output directory.
+ */
+const getFileExtension = (contentType) => {
+    switch (contentType) {
+        case 'sqlChart':
+            return '.sql.yml';
+        case 'chart':
+        case 'dashboard':
+        default:
+            return '.yml';
+    }
+};
+const writeContent = async (contentAsCode, outputDir, languageMap) => {
+    const extension = getFileExtension(contentAsCode.type);
+    const itemPath = path.join(outputDir, `${contentAsCode.content.slug}${extension}`);
+    // Strip timestamps — they go to .clary-metadata.json instead
+    const { updatedAt, downloadedAt, ...cleanContent } = contentAsCode.content;
+    const chartYml = yaml.dump(cleanContent, {
+        quotingType: '"',
+        sortKeys: true,
+    });
+    await fs_1.promises.writeFile(itemPath, chartYml);
+    if (contentAsCode.translationMap && languageMap) {
+        const translationPath = path.join(outputDir, `${contentAsCode.content.slug}.language.map.yml`);
+        await fs_1.promises.writeFile(translationPath, yaml.dump(contentAsCode.translationMap, { sortKeys: true }));
+    }
+    const metadataType = contentAsCode.type === 'dashboard' ? 'dashboards' : 'charts';
+    let downloadedAtString;
+    if (downloadedAt instanceof Date) {
+        downloadedAtString = downloadedAt.toISOString();
+    }
+    else if (typeof downloadedAt === 'string') {
+        downloadedAtString = downloadedAt;
+    }
+    else {
+        downloadedAtString = new Date().toISOString();
+    }
+    return {
+        slug: contentAsCode.content.slug,
+        type: metadataType,
+        downloadedAt: downloadedAtString,
+    };
+};
+/**
+ * Writes space YAML files for each space in the download results.
+ * In flat mode, files go in the base download directory.
+ * In nested mode, files go in the root of each space's directory.
+ */
+const writeSpaceFiles = async (spaces, projectName, customPath, folderScheme = 'flat') => {
+    if (spaces.length === 0)
+        return;
+    const baseDir = getDownloadFolder(customPath);
+    const seen = new Set();
+    for (const space of spaces) {
+        // Deduplicate across paginated API responses
+        if (!seen.has(space.slug)) {
+            seen.add(space.slug);
+            let outputDir;
+            if (folderScheme === 'nested') {
+                outputDir = path.join(baseDir, projectName, space.slug);
+            }
+            else {
+                outputDir = baseDir;
+            }
+            await fs_1.promises.mkdir(outputDir, { recursive: true });
+            const fileName = `${(0, common_1.generateSlug)(space.spaceName)}.space.yml`;
+            const filePath = path.join(outputDir, fileName);
+            const content = yaml.dump(space, {
+                quotingType: '"',
+                sortKeys: true,
+            });
+            await fs_1.promises.writeFile(filePath, content);
+            globalState_1.default.debug(`Wrote space file: ${filePath}`);
+        }
+    }
+};
+/**
+ * Reads all .space.yml files from the download directory and returns
+ * a map of space slug → original space name. Used during upload to
+ * preserve human-readable space names instead of deriving them from slugs.
+ */
+const readSpaceNames = async (customPath) => {
+    const baseDir = getDownloadFolder(customPath);
+    const spaceNames = {};
+    try {
+        const allEntries = await fs_1.promises.readdir(baseDir, {
+            recursive: true,
+            withFileTypes: true,
+        });
+        await Promise.all(allEntries
+            .filter((entry) => entry.isFile() && entry.name.endsWith('.space.yml'))
+            .map(async (file) => {
+            try {
+                const filePath = path.join(file.parentPath, file.name);
+                const fileContent = await fs_1.promises.readFile(filePath, 'utf-8');
+                const parsed = yaml.load(fileContent);
+                if (parsed?.contentType ===
+                    common_1.ContentAsCodeType.SPACE &&
+                    typeof parsed.slug === 'string' &&
+                    typeof parsed.spaceName === 'string') {
+                    spaceNames[parsed.slug] = parsed.spaceName;
+                }
+            }
+            catch (e) {
+                globalState_1.default.debug(`Skipping space file ${file.name}: ${(0, common_1.getErrorMessage)(e)}`);
+            }
+        }));
+    }
+    catch {
+        // Directory doesn't exist or can't be read — return empty map
+    }
+    return spaceNames;
+};
+const hasUnsortedKeys = (obj) => {
+    if (typeof obj !== 'object' || obj === null || Array.isArray(obj)) {
+        if (Array.isArray(obj)) {
+            return obj.some(hasUnsortedKeys);
+        }
+        return false;
+    }
+    const keys = Object.keys(obj);
+    const sorted = [...keys].sort();
+    if (keys.some((key, i) => key !== sorted[i])) {
+        return true;
+    }
+    return Object.values(obj).some(hasUnsortedKeys);
+};
+const isLightdashContentFile = (folder, entry) => entry.isFile() &&
+    entry.parentPath &&
+    entry.parentPath.endsWith(path.sep + folder) &&
+    entry.name.endsWith('.yml') &&
+    !entry.name.endsWith('.language.map.yml');
+const isLooseContentFile = (entry) => entry.isFile() &&
+    entry.parentPath &&
+    !entry.parentPath.endsWith(`${path.sep}charts`) &&
+    !entry.parentPath.endsWith(`${path.sep}dashboards`) &&
+    entry.name.endsWith('.yml') &&
+    !entry.name.endsWith('.language.map.yml');
+const processYamlItem = (item, fileName, stats, folder, metadata) => {
+    if (hasUnsortedKeys(item)) {
+        globalState_1.default.log(styles.warning(`Warning: ${fileName} has unsorted YAML keys. Re-download to fix, or sort keys alphabetically.`));
+    }
+    const metadataSection = folder === 'dashboards' ? metadata.dashboards : metadata.charts;
+    const downloadedAtRaw = metadataSection[item.slug] ?? item.downloadedAt;
+    const downloadedAt = downloadedAtRaw
+        ? new Date(downloadedAtRaw instanceof Date
+            ? downloadedAtRaw.getTime()
+            : downloadedAtRaw)
+        : undefined;
+    const needsUpdating = downloadedAt &&
+        Math.abs(stats.mtime.getTime() - downloadedAt.getTime()) > 30000;
+    return {
+        ...item,
+        updatedAt: needsUpdating ? stats.mtime : item.updatedAt,
+        needsUpdating: needsUpdating ?? true,
+    };
+};
+const loadYamlFile = async (file, folder, metadata) => {
+    const filePath = path.join(file.parentPath, file.name);
+    const [fileContent, stats] = await Promise.all([
+        fs_1.promises.readFile(filePath, 'utf-8'),
+        fs_1.promises.stat(filePath),
+    ]);
+    const item = yaml.load(fileContent);
+    return processYamlItem(item, file.name, stats, folder, metadata);
+};
+const readCodeFiles = async (folder, customPath) => {
+    const baseDir = getDownloadFolder(customPath);
+    globalState_1.default.log(`Reading ${folder} from ${baseDir}`);
+    const [major, minor] = process.versions.node.split('.').map(Number);
+    if (major < 20 || (major === 20 && minor < 12)) {
+        throw new Error(`Node.js v20.12.0 or later is required for this command (current: ${process.version}).`);
+    }
+    try {
+        const metadata = await (0, metadataFile_1.readMetadataFile)(baseDir);
+        const allEntries = await fs_1.promises.readdir(baseDir, {
+            recursive: true,
+            withFileTypes: true,
+        });
+        const items = await Promise.all(allEntries
+            .filter((entry) => isLightdashContentFile(folder, entry))
+            .map((file) => loadYamlFile(file, folder, metadata)));
+        if (items.length === 0) {
+            console.error(styles.warning(`Unable to upload ${folder}, no files found in "${baseDir}". Run download command first.`));
+        }
+        return items;
+    }
+    catch (error) {
+        // Handle case where base directory doesn't exist
+        if (error.code === 'ENOENT') {
+            console.error(styles.warning(`Unable to upload ${folder}, "${baseDir}" folder not found. Run download command first.`));
+            return [];
+        }
+        // Unknown error
+        console.error(styles.error(`Error reading ${baseDir}: ${error}`));
+        throw error;
+    }
+};
+/**
+ * Reads YAML files outside the standard charts/ and dashboards/ directories
+ * and classifies them by their contentType field.
+ */
+const readLooseCodeFiles = async (customPath) => {
+    const baseDir = getDownloadFolder(customPath);
+    const charts = [];
+    const dashboards = [];
+    try {
+        const metadata = await (0, metadataFile_1.readMetadataFile)(baseDir);
+        const allEntries = await fs_1.promises.readdir(baseDir, {
+            recursive: true,
+            withFileTypes: true,
+        });
+        const looseFiles = allEntries.filter(isLooseContentFile);
+        await Promise.all(looseFiles.map(async (file) => {
+            try {
+                const filePath = path.join(file.parentPath, file.name);
+                const [fileContent, stats] = await Promise.all([
+                    fs_1.promises.readFile(filePath, 'utf-8'),
+                    fs_1.promises.stat(filePath),
+                ]);
+                const parsed = yaml.load(fileContent);
+                const contentType = parsed?.contentType;
+                if (contentType === common_1.ContentAsCodeType.CHART ||
+                    contentType === common_1.ContentAsCodeType.SQL_CHART) {
+                    charts.push(processYamlItem(parsed, file.name, stats, 'charts', metadata));
+                }
+                else if (contentType === common_1.ContentAsCodeType.DASHBOARD) {
+                    dashboards.push(processYamlItem(parsed, file.name, stats, 'dashboards', metadata));
+                }
+                else if (contentType === common_1.ContentAsCodeType.SPACE) {
+                    // Space YAML files are metadata-only until the next PR
+                }
+                else {
+                    globalState_1.default.debug(`Skipping ${file.name}: no recognized contentType`);
+                }
+            }
+            catch (e) {
+                globalState_1.default.debug(`Skipping ${file.name}: failed to parse (${(0, common_1.getErrorMessage)(e)})`);
+            }
+        }));
+    }
+    catch (error) {
+        if (error.code === 'ENOENT') {
+            // Base directory doesn't exist — nothing to discover
+            return { charts, dashboards };
+        }
+        throw error;
+    }
+    return { charts, dashboards };
+};
+const groupBySpace = (items) => {
+    const itemsWithIndex = items.map((item, index) => ({ item, index }));
+    return (0, groupBy_1.default)(itemsWithIndex, (entry) => entry.item.spaceSlug);
+};
+const writeSpaceContent = async ({ projectName, spaceSlug, folder, contentType, contentInSpace, contentAsCode, customPath, languageMap, folderScheme, }) => {
+    const outputDir = await createDirForContent(projectName, spaceSlug, folder, customPath, folderScheme);
+    const entries = [];
+    for (const { item, index } of contentInSpace) {
+        const translationMap = 'languageMap' in contentAsCode
+            ? contentAsCode.languageMap?.[index]
+            : undefined;
+        const entry = await writeContent({
+            type: contentType,
+            content: item,
+            translationMap,
+        }, outputDir, languageMap);
+        entries.push(entry);
+    }
+    return entries;
+};
+const getContentTypeConfig = (type, projectId) => {
+    switch (type) {
+        case 'charts':
+            return {
+                endpoint: `/api/v1/projects/${projectId}/charts/code`,
+                displayName: 'charts',
+                supportsLanguageMap: true,
+            };
+        case 'dashboards':
+            return {
+                endpoint: `/api/v1/projects/${projectId}/dashboards/code`,
+                displayName: 'dashboards',
+                supportsLanguageMap: true,
+            };
+        case 'sqlCharts':
+            return {
+                endpoint: `/api/v1/projects/${projectId}/sqlCharts/code`,
+                displayName: 'SQL charts',
+                supportsLanguageMap: false,
+            };
+        default:
+            return (0, common_1.assertUnreachable)(type, `Unknown content type: ${type}`);
+    }
+};
+const extractChartSlugsFromDashboards = (dashboards) => dashboards.reduce((acc, dashboard) => {
+    const slugs = dashboard.tiles
+        .map((tile) => 'chartSlug' in tile.properties
+        ? tile.properties.chartSlug
+        : undefined)
+        .filter((slug) => slug !== undefined);
+    return [...acc, ...slugs];
+}, []);
+const downloadContent = async (ids, type, projectId, projectName, customPath, languageMap = false, nested = false, skipSpaces = false) => {
+    const spinner = globalState_1.default.getActiveSpinner();
+    const contentFilters = parseContentFilters(ids);
+    const folderScheme = nested ? 'nested' : 'flat';
+    const config = getContentTypeConfig(type, projectId);
+    let offset = 0;
+    let total = 0;
+    let chartSlugs = [];
+    let allMetadataEntries = [];
+    let allSpaces = [];
+    do {
+        globalState_1.default.debug(`Downloading ${config.displayName} with offset "${offset}" and filters "${contentFilters}"`);
+        const commonParams = config.supportsLanguageMap
+            ? `offset=${offset}&languageMap=${languageMap}`
+            : `offset=${offset}`;
+        const queryParams = contentFilters
+            ? `${contentFilters}&${commonParams}`
+            : `?${commonParams}`;
+        const results = await (0, apiClient_1.lightdashApi)({
+            method: 'GET',
+            url: `${config.endpoint}${queryParams}`,
+            body: undefined,
+        });
+        spinner?.start(`Downloaded ${results.offset} of ${results.total} ${config.displayName}`);
+        // For the same chart slug, we run the code for saved charts and sql chart
+        // so we are going to get more false positives here, so we keep it on the debug log
+        results.missingIds.forEach((missingId) => {
+            globalState_1.default.debug(`\nNo ${config.displayName} with id "${missingId}"`);
+        });
+        // Write content based on type
+        if ('sqlCharts' in results) {
+            const sqlChartsBySpace = groupBySpace(results.sqlCharts);
+            for (const [spaceSlug, sqlChartsInSpace] of Object.entries(sqlChartsBySpace)) {
+                const entries = await writeSpaceContent({
+                    projectName,
+                    spaceSlug,
+                    folder: 'charts',
+                    contentType: 'sqlChart',
+                    contentInSpace: sqlChartsInSpace,
+                    contentAsCode: results,
+                    customPath,
+                    languageMap,
+                    folderScheme,
+                });
+                allMetadataEntries = [...allMetadataEntries, ...entries];
+            }
+        }
+        else if ('dashboards' in results) {
+            const dashboardsBySpace = groupBySpace(results.dashboards);
+            for (const [spaceSlug, dashboardsInSpace] of Object.entries(dashboardsBySpace)) {
+                const entries = await writeSpaceContent({
+                    projectName,
+                    spaceSlug,
+                    folder: 'dashboards',
+                    contentType: 'dashboard',
+                    contentInSpace: dashboardsInSpace,
+                    contentAsCode: results,
+                    customPath,
+                    languageMap,
+                    folderScheme,
+                });
+                allMetadataEntries = [...allMetadataEntries, ...entries];
+            }
+            chartSlugs = [
+                ...chartSlugs,
+                ...extractChartSlugsFromDashboards(results.dashboards),
+            ];
+        }
+        else {
+            const chartsBySpace = groupBySpace(results.charts);
+            for (const [spaceSlug, chartsInSpace] of Object.entries(chartsBySpace)) {
+                const entries = await writeSpaceContent({
+                    projectName,
+                    spaceSlug,
+                    folder: 'charts',
+                    contentType: 'chart',
+                    contentInSpace: chartsInSpace,
+                    contentAsCode: results,
+                    customPath,
+                    languageMap,
+                    folderScheme,
+                });
+                allMetadataEntries = [...allMetadataEntries, ...entries];
+            }
+        }
+        // Accumulate space metadata from each page
+        if ('spaces' in results && results.spaces) {
+            allSpaces = [...allSpaces, ...results.spaces];
+        }
+        offset = results.offset;
+        total = results.total;
+    } while (offset < total);
+    // Write space YAML files
+    if (!skipSpaces) {
+        await writeSpaceFiles(allSpaces, projectName, customPath, folderScheme);
+    }
+    return [total, [...new Set(chartSlugs)], allMetadataEntries, allSpaces];
+};
+exports.downloadContent = downloadContent;
+const downloadHandler = async (options) => {
+    globalState_1.default.setVerbose(options.verbose);
+    await (0, apiClient_1.checkLightdashVersion)();
+    const config = await (0, config_1.getConfig)();
+    if (!config.context?.apiKey || !config.context.serverUrl) {
+        throw new common_1.AuthorizationError(`Not logged in. Run 'clary login --help'`);
+    }
+    const projectSelection = await (0, selectProject_1.selectProject)(config, options.project);
+    if (!projectSelection) {
+        throw new common_1.LightdashError({
+            message: 'No project selected. Run clary config set-project',
+            name: 'Not Found',
+            statusCode: 404,
+            data: {},
+        });
+    }
+    const projectId = projectSelection.projectUuid;
+    // Log current project info
+    (0, selectProject_1.logSelectedProject)(projectSelection, config, 'Downloading from');
+    const spinner = globalState_1.default.startSpinner(`Downloading charts`);
+    spinner.start(`Downloading content from project`);
+    // Fetch project details to get project name for folder structure
+    const project = await (0, apiClient_1.lightdashApi)({
+        method: 'GET',
+        url: `/api/v1/projects/${projectId}`,
+        body: undefined,
+    });
+    const projectName = (0, common_1.generateSlug)(project.name);
+    // For analytics
+    let chartTotal;
+    let dashboardTotal;
+    const start = Date.now();
+    await analytics_1.LightdashAnalytics.track({
+        event: 'download.started',
+        properties: {
+            userId: config.user?.userUuid,
+            organizationId: config.user?.organizationUuid,
+            projectId,
+        },
+    });
+    try {
+        const hasFilters = options.charts.length > 0 || options.dashboards.length > 0;
+        // When downloading specific charts or dashboards, skip space metadata
+        const skipSpaces = options.skipSpaces || hasFilters;
+        let allMetadataEntries = [];
+        let allSpaces = [];
+        // Download regular charts
+        if (hasFilters && options.charts.length === 0) {
+            console.info(styles.warning(`No charts filters provided, skipping`));
+        }
+        else {
+            const [regularChartTotal, , regularChartMeta, regularChartSpaces] = await (0, exports.downloadContent)(options.charts, 'charts', projectId, projectName, options.path, options.languageMap, options.nested, skipSpaces);
+            spinner.succeed(`Downloaded ${regularChartTotal} charts`);
+            allMetadataEntries = [...allMetadataEntries, ...regularChartMeta];
+            allSpaces = [...allSpaces, ...regularChartSpaces];
+            // Download SQL charts
+            spinner.start(`Downloading SQL charts`);
+            const [sqlChartTotal, , sqlChartMeta, sqlChartSpaces] = await (0, exports.downloadContent)(options.charts, 'sqlCharts', projectId, projectName, options.path, options.languageMap, options.nested, skipSpaces);
+            spinner.succeed(`Downloaded ${sqlChartTotal} SQL charts`);
+            allMetadataEntries = [...allMetadataEntries, ...sqlChartMeta];
+            allSpaces = [...allSpaces, ...sqlChartSpaces];
+            chartTotal = regularChartTotal + sqlChartTotal;
+        }
+        // Download dashboards
+        if (hasFilters && options.dashboards.length === 0) {
+            console.info(styles.warning(`No dashboards filters provided, skipping`));
+        }
+        else {
+            let chartSlugs = [];
+            let dashMeta;
+            let dashSpaces;
+            [dashboardTotal, chartSlugs, dashMeta, dashSpaces] =
+                await (0, exports.downloadContent)(options.dashboards, 'dashboards', projectId, projectName, options.path, options.languageMap, options.nested, skipSpaces);
+            allMetadataEntries = [...allMetadataEntries, ...dashMeta];
+            allSpaces = [...allSpaces, ...dashSpaces];
+            spinner.succeed(`Downloaded ${dashboardTotal} dashboards`);
+            if (hasFilters && chartSlugs.length > 0) {
+                spinner.start(`Downloading ${chartSlugs.length} charts linked to dashboards`);
+                const [regularCharts, , linkedChartMeta, linkedChartSpaces] = await (0, exports.downloadContent)(chartSlugs, 'charts', projectId, projectName, options.path, options.languageMap, options.nested, skipSpaces);
+                allMetadataEntries = [
+                    ...allMetadataEntries,
+                    ...linkedChartMeta,
+                ];
+                allSpaces = [...allSpaces, ...linkedChartSpaces];
+                const [sqlCharts, , linkedSqlMeta, linkedSqlSpaces] = await (0, exports.downloadContent)(chartSlugs, 'sqlCharts', projectId, projectName, options.path, options.languageMap, options.nested, skipSpaces);
+                allMetadataEntries = [...allMetadataEntries, ...linkedSqlMeta];
+                allSpaces = [...allSpaces, ...linkedSqlSpaces];
+                spinner.succeed(`Downloaded ${regularCharts + sqlCharts} charts linked to dashboards`);
+            }
+        }
+        // Report space definitions count
+        if (!skipSpaces) {
+            const uniqueSpaceCount = new Set(allSpaces.map((s) => s.slug)).size;
+            spinner.succeed(`Downloaded ${uniqueSpaceCount} space definitions`);
+        }
+        // Write metadata file with all downloadedAt timestamps
+        const metadataToWrite = {
+            version: 1,
+            charts: {},
+            dashboards: {},
+        };
+        for (const entry of allMetadataEntries) {
+            metadataToWrite[entry.type][entry.slug] = entry.downloadedAt;
+        }
+        const baseDir = getDownloadFolder(options.path);
+        const downloadRoot = options.nested
+            ? path.join(baseDir, projectName)
+            : baseDir;
+        await (0, metadataFile_1.writeMetadataFile)(baseDir, metadataToWrite);
+        if (!config.answers?.metadataFileGitignoreNoticeShown) {
+            globalState_1.default.log(styles.warning(`\nNote: ${metadataFile_1.METADATA_FILENAME} was written to ${baseDir}. Consider adding it to your .gitignore.`));
+            await (0, config_1.setAnswer)({ metadataFileGitignoreNoticeShown: true });
+        }
+        globalState_1.default.log(styles.success(`Downloaded content saved to ${downloadRoot}`));
+        const end = Date.now();
+        await analytics_1.LightdashAnalytics.track({
+            event: 'download.completed',
+            properties: {
+                userId: config.user?.userUuid,
+                organizationId: config.user?.organizationUuid,
+                projectId,
+                chartsNum: chartTotal,
+                dashboardsNum: dashboardTotal,
+                timeToCompleted: (end - start) / 1000,
+            },
+        });
+    }
+    catch (error) {
+        console.error(styles.error(`\nError downloading ${error}`));
+        await analytics_1.LightdashAnalytics.track({
+            event: 'download.error',
+            properties: {
+                userId: config.user?.userUuid,
+                organizationId: config.user?.organizationUuid,
+                projectId,
+                error: `${error}`,
+            },
+        });
+    }
+};
+exports.downloadHandler = downloadHandler;
+const getPromoteAction = (action) => {
+    switch (action) {
+        case common_1.PromotionAction.CREATE:
+            return 'created';
+        case common_1.PromotionAction.UPDATE:
+            return 'updated';
+        case common_1.PromotionAction.DELETE:
+            return 'deleted';
+        case common_1.PromotionAction.NO_CHANGES:
+            return 'skipped';
+        default:
+            (0, common_1.assertUnreachable)(action, `Unknown promotion action: ${action}`);
+    }
+    return 'skipped';
+};
+const storeUploadChanges = (changes, promoteChanges) => {
+    const getPromoteChanges = (resource) => {
+        const promotions = promoteChanges[resource];
+        return promotions.reduce((acc, promoteChange) => {
+            const action = getPromoteAction(promoteChange.action);
+            const key = `${resource} ${action}`;
+            acc[key] = (acc[key] ?? 0) + 1;
+            return acc;
+        }, {});
+    };
+    const updatedChanges = {
+        ...changes,
+    };
+    ['spaces', 'charts', 'dashboards'].forEach((resource) => {
+        const resourceChanges = getPromoteChanges(resource);
+        Object.entries(resourceChanges).forEach(([key, value]) => {
+            updatedChanges[key] = (updatedChanges[key] ?? 0) + value;
+        });
+    });
+    return updatedChanges;
+};
+const logUploadChanges = (changes) => {
+    Object.entries(changes).forEach(([key, value]) => {
+        console.info(`Total ${key}: ${value} `);
+    });
+    const totalSkipped = Object.entries(changes)
+        .filter(([key]) => key.includes('skipped'))
+        .reduce((sum, [, value]) => sum + value, 0);
+    const totalUpserted = Object.entries(changes)
+        .filter(([key]) => !key.includes('skipped'))
+        .reduce((sum, [, value]) => sum + value, 0);
+    if (totalSkipped > 0 && totalUpserted === 0) {
+        console.warn(styles.warning(`\nAll content was skipped (no local changes detected). Use --force to upload all content, e.g. when uploading to a new project.`));
+    }
+};
+// SQL charts have 'sql' field instead of 'tableName'/'metricQuery'
+const isSqlChart = (item) => 'sql' in item && !('tableName' in item);
+const upsertSingleItem = async (item, type, projectId, changes, force, config, skipSpaceCreate, publicSpaceCreate, validate, spaceNames) => {
+    try {
+        if (!force && !item.needsUpdating) {
+            globalState_1.default.debug(`Skipping ${type} "${item.slug}" with no local changes`);
+            changes[`${type} skipped`] = (changes[`${type} skipped`] ?? 0) + 1;
+            return;
+        }
+        globalState_1.default.debug(`Upserting ${type} ${item.slug}`);
+        // SQL charts use a different endpoint
+        const isSqlChartItem = type === 'charts' && isSqlChart(item);
+        const endpoint = isSqlChartItem
+            ? `/api/v1/projects/${projectId}/sqlCharts/${item.slug}/code`
+            : `/api/v1/projects/${projectId}/${type}/${item.slug}/code`;
+        const upsertData = await (0, apiClient_1.lightdashApi)({
+            method: 'POST',
+            url: endpoint,
+            body: JSON.stringify({
+                ...item,
+                skipSpaceCreate,
+                publicSpaceCreate,
+                force,
+                ...(spaceNames &&
+                    Object.keys(spaceNames).length > 0 && { spaceNames }),
+            }),
+        });
+        globalState_1.default.debug(`${type} "${item.name}": ${upsertData[type]?.[0].action}`);
+        // Merge storeUploadChanges result into changes in-place
+        const updatedChanges = storeUploadChanges(changes, upsertData);
+        Object.keys(updatedChanges).forEach((key) => {
+            changes[key] = updatedChanges[key];
+        });
+        // Warn if contentType contradicts the folder this item came from
+        const itemContentType = item.contentType;
+        if (itemContentType) {
+            const expectedType = itemContentType === common_1.ContentAsCodeType.DASHBOARD
+                ? 'dashboards'
+                : 'charts';
+            if (expectedType !== type) {
+                globalState_1.default.log(styles.warning(`Warning: "${item.name}" has contentType "${itemContentType}" but is in the ${type}/ directory. It will be uploaded as a ${type.slice(0, -1)}.`));
+            }
+        }
+        // Run validation if requested
+        if (validate && !isSqlChartItem) {
+            const contentUuid = type === 'charts'
+                ? upsertData.charts?.[0]?.data?.uuid
+                : upsertData.dashboards?.[0]?.data?.uuid;
+            if (contentUuid) {
+                try {
+                    const validationEndpoint = type === 'charts'
+                        ? `/api/v1/projects/${projectId}/validate/chart/${contentUuid}`
+                        : `/api/v1/projects/${projectId}/validate/dashboard/${contentUuid}`;
+                    const validationResult = await (0, apiClient_1.lightdashApi)({
+                        method: 'POST',
+                        url: validationEndpoint,
+                        body: JSON.stringify({}),
+                    });
+                    if (validationResult.errors &&
+                        validationResult.errors.length > 0) {
+                        globalState_1.default.log(styles.warning(`Validation found ${validationResult.errors.length} issue(s) in ${type.slice(0, -1)} "${item.name}"`));
+                        validationResult.errors.forEach((error) => {
+                            globalState_1.default.log(styles.warning(`  - ${error.error}`));
+                        });
+                    }
+                    else {
+                        globalState_1.default.log(styles.success(`✓ No validation issues in ${type.slice(0, -1)} "${item.name}"`));
+                    }
+                }
+                catch (validationError) {
+                    globalState_1.default.debug(`Validation failed for ${type.slice(0, -1)} "${item.name}": ${(0, common_1.getErrorMessage)(validationError)}`);
+                }
+            }
+        }
+    }
+    catch (error) {
+        if (error instanceof common_1.LightdashError &&
+            error.name === 'NotFoundError' &&
+            skipSpaceCreate) {
+            globalState_1.default.log(styles.warning(`Skipping ${type} "${item.slug}" because space "${item.spaceSlug}" does not exist and --skip-space-create is true`));
+            changes[`${type} skipped`] = (changes[`${type} skipped`] ?? 0) + 1;
+        }
+        else {
+            changes[`${type} with errors`] =
+                (changes[`${type} with errors`] ?? 0) + 1;
+            globalState_1.default.log(styles.error(`Error upserting ${type}:\n\t"${item.name}" (slug: "${item.slug}")\n\t${(0, common_1.getErrorMessage)(error)}`));
+            await analytics_1.LightdashAnalytics.track({
+                event: 'download.error',
+                properties: {
+                    userId: config.user?.userUuid,
+                    organizationId: config.user?.organizationUuid,
+                    projectId,
+                    type,
+                    error: (0, common_1.getErrorMessage)(error),
+                },
+            });
+        }
+    }
+};
+/**
+ *
+ * @param slugs if slugs are provided, we only force upsert the charts/dashboards that match the slugs, if slugs are empty, we upload files that were locally updated
+ */
+const upsertResources = async (type, projectId, changes, force, slugs, customPath, skipSpaceCreate, publicSpaceCreate, validate, concurrency = 1, extraItems = [], spaceNames) => {
+    const config = await (0, config_1.getConfig)();
+    const folderItems = await readCodeFiles(type, customPath);
+    const items = [...folderItems, ...extraItems];
+    globalState_1.default.log(`Found ${items.length} ${type} files`);
+    const hasFilter = slugs.length > 0;
+    const filteredItems = hasFilter
+        ? items.filter((item) => slugs.includes(item.slug))
+        : items;
+    if (hasFilter) {
+        globalState_1.default.log(`Filtered ${filteredItems.length} ${type} with slugs: ${slugs.join(', ')}`);
+        const missingItems = slugs.filter((slug) => !items.find((item) => item.slug === slug));
+        missingItems.forEach((slug) => {
+            globalState_1.default.log(styles.warning(`No ${type} with slug: "${slug}"`));
+        });
+    }
+    if (concurrency <= 1) {
+        // Sequential path — preserves original behavior exactly
+        for (const item of filteredItems) {
+            // eslint-disable-next-line no-await-in-loop
+            await upsertSingleItem(item, type, projectId, changes, force, config, skipSpaceCreate, publicSpaceCreate, validate, spaceNames);
+        }
+    }
+    else {
+        const grouped = (0, groupBy_1.default)(filteredItems, (item) => item.spaceSlug);
+        const seedItems = new Set();
+        const remainingItems = [];
+        Object.values(grouped).forEach((spaceItems) => {
+            // Pick the first item that will actually trigger an API call
+            // (and thus create the space). If force is true, any item works.
+            const seedIndex = force
+                ? 0
+                : spaceItems.findIndex((i) => i.needsUpdating);
+            if (seedIndex >= 0) {
+                seedItems.add(spaceItems[seedIndex]);
+                remainingItems.push(...spaceItems.filter((_, idx) => idx !== seedIndex));
+            }
+            else {
+                // No items need updating — all will be skipped, no space needed
+                remainingItems.push(...spaceItems);
+            }
+        });
+        // For charts: also seed one item per unique dashboardSlug to avoid
+        // concurrent placeholder dashboard creation (duplicate slug bug)
+        if (type === 'charts') {
+            const chartsWithDashboard = remainingItems.filter((item) => 'dashboardSlug' in item &&
+                item.dashboardSlug);
+            const groupedByDashboard = (0, groupBy_1.default)(chartsWithDashboard, (item) => item.dashboardSlug);
+            Object.values(groupedByDashboard).forEach((dashboardItems) => {
+                // If no item for this dashboardSlug was already picked as a
+                // space seed, pick the first one as a dashboard seed
+                const alreadySeeded = dashboardItems.some((item) => seedItems.has(item));
+                if (!alreadySeeded) {
+                    const seedIndex = force
+                        ? 0
+                        : dashboardItems.findIndex((i) => i.needsUpdating);
+                    if (seedIndex >= 0) {
+                        seedItems.add(dashboardItems[seedIndex]);
+                        // Remove from remainingItems since it's now a seed
+                        const idx = remainingItems.indexOf(dashboardItems[seedIndex]);
+                        if (idx >= 0) {
+                            remainingItems.splice(idx, 1);
+                        }
+                    }
+                }
+            });
+        }
+        // Phase 1: Sequential seeding (spaces + dashboard placeholders)
+        for (const item of seedItems) {
+            // eslint-disable-next-line no-await-in-loop
+            await upsertSingleItem(item, type, projectId, changes, force, config, skipSpaceCreate, publicSpaceCreate, validate, spaceNames);
+        }
+        // Phase 2: Parallel bulk upload of remaining items
+        const limit = (0, p_limit_1.default)(concurrency);
+        await Promise.all(remainingItems.map((item) => limit(async () => {
+            await upsertSingleItem(item, type, projectId, changes, force, config, skipSpaceCreate, publicSpaceCreate, validate, spaceNames);
+        })));
+    }
+    return { changes, total: filteredItems.length };
+};
+const getDashboardChartSlugs = async (dashboardSlugs, customPath) => {
+    const dashboardItems = await readCodeFiles('dashboards', customPath);
+    const filteredDashboardItems = dashboardSlugs.length > 0
+        ? dashboardItems.filter((dashboard) => dashboardSlugs.includes(dashboard.slug))
+        : dashboardItems;
+    return filteredDashboardItems.reduce((acc, dashboard) => {
+        const dashboardChartSlugs = dashboard.tiles
+            .map((tile) => 'chartSlug' in tile.properties
+            ? tile.properties.chartSlug
+            : undefined)
+            .filter((dashboardChartSlug) => !!dashboardChartSlug);
+        return [...acc, ...dashboardChartSlugs];
+    }, []);
+};
+const uploadHandler = async (options) => {
+    globalState_1.default.setVerbose(options.verbose);
+    if (options.gzip) {
+        (0, apiClient_1.setGzipEnabled)(true);
+    }
+    await (0, apiClient_1.checkLightdashVersion)();
+    const config = await (0, config_1.getConfig)();
+    if (!config.context?.apiKey || !config.context.serverUrl) {
+        throw new common_1.AuthorizationError(`Not logged in. Run 'clary login --help'`);
+    }
+    const projectSelection = await (0, selectProject_1.selectProject)(config, options.project);
+    if (!projectSelection) {
+        throw new common_1.LightdashError({
+            message: 'No project selected. Run clary config set-project',
+            name: 'Not Found',
+            statusCode: 404,
+            data: {},
+        });
+    }
+    const projectId = projectSelection.projectUuid;
+    // Log current project info
+    (0, selectProject_1.logSelectedProject)(projectSelection, config, 'Uploading to');
+    let changes = {};
+    // For analytics
+    let chartTotal;
+    let dashboardTotal;
+    const start = Date.now();
+    await analytics_1.LightdashAnalytics.track({
+        event: 'upload.started',
+        properties: {
+            userId: config.user?.userUuid,
+            organizationId: config.user?.organizationUuid,
+            projectId,
+        },
+    });
+    try {
+        // If any filter is provided, we skip those items without filters
+        // eg: if a --charts filter is provided, we skip dashboards if no --dashboards filter is provided
+        const hasFilters = options.charts.length > 0 || options.dashboards.length > 0;
+        // Always include the charts from dashboards if includeCharts is true regardless of the charts filters
+        const chartSlugs = options.includeCharts
+            ? Array.from(new Set([
+                ...options.charts,
+                ...(await getDashboardChartSlugs(options.dashboards, options.path)),
+            ]))
+            : options.charts;
+        const concurrency = Math.min(Math.max(1, parseInt(String(options.concurrency), 10) || 1), 1000);
+        if (parseInt(String(options.concurrency), 10) > 1000) {
+            globalState_1.default.log(styles.warning(`Concurrency limit exceeded. Using maximum of 1000 instead of ${options.concurrency}`));
+        }
+        // Read space definition files to preserve original space names during upload
+        const spaceNames = await readSpaceNames(options.path);
+        if (Object.keys(spaceNames).length > 0) {
+            globalState_1.default.log(`Found ${Object.keys(spaceNames).length} space definition(s)`);
+        }
+        // Discover loose YAML files (outside charts/ and dashboards/) classified by contentType
+        const looseFiles = await readLooseCodeFiles(options.path);
+        if (looseFiles.charts.length > 0) {
+            globalState_1.default.log(`Found ${looseFiles.charts.length} chart(s) outside charts/ directory (classified by contentType)`);
+        }
+        if (looseFiles.dashboards.length > 0) {
+            globalState_1.default.log(`Found ${looseFiles.dashboards.length} dashboard(s) outside dashboards/ directory (classified by contentType)`);
+        }
+        if (hasFilters && chartSlugs.length === 0) {
+            globalState_1.default.log(styles.warning(`No charts filters provided, skipping`));
+        }
+        else {
+            const { changes: chartChanges, total } = await upsertResources('charts', projectId, changes, options.force, chartSlugs, options.path, options.skipSpaceCreate, options.public, options.validate, concurrency, looseFiles.charts, spaceNames);
+            changes = chartChanges;
+            chartTotal = total;
+        }
+        if (hasFilters && options.dashboards.length === 0) {
+            globalState_1.default.log(styles.warning(`No dashboard filters provided, skipping`));
+        }
+        else {
+            const { changes: dashboardChanges, total } = await upsertResources('dashboards', projectId, changes, options.force, options.dashboards, options.path, options.skipSpaceCreate, options.public, options.validate, concurrency, looseFiles.dashboards, spaceNames);
+            changes = dashboardChanges;
+            dashboardTotal = total;
+        }
+        const end = Date.now();
+        await analytics_1.LightdashAnalytics.track({
+            event: 'upload.completed',
+            properties: {
+                userId: config.user?.userUuid,
+                organizationId: config.user?.organizationUuid,
+                projectId,
+                chartsNum: chartTotal,
+                dashboardsNum: dashboardTotal,
+                timeToCompleted: (end - start) / 1000, // in seconds
+            },
+        });
+        logUploadChanges(changes);
+    }
+    catch (error) {
+        globalState_1.default.log(styles.error(`\nError downloading: ${(0, common_1.getErrorMessage)(error)}`));
+        await analytics_1.LightdashAnalytics.track({
+            event: 'download.error',
+            properties: {
+                userId: config.user?.userUuid,
+                organizationId: config.user?.organizationUuid,
+                projectId,
+                error: (0, common_1.getErrorMessage)(error),
+            },
+        });
+    }
+};
+exports.uploadHandler = uploadHandler;