@lightdash/cli 0.3399.1 → 0.3400.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/handlers/download.d.ts +14 -1
- package/dist/handlers/download.d.ts.map +1 -1
- package/dist/handlers/download.js +155 -145
- package/dist/handlers/download.test.js +474 -1
- package/dist/handlers/spacesAsCode.d.ts +29 -0
- package/dist/handlers/spacesAsCode.d.ts.map +1 -0
- package/dist/handlers/spacesAsCode.js +603 -0
- package/dist/index.js +5 -1
- package/package.json +3 -3
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { AgentAsCode, ChartAsCode, DashboardAsCode, type SpaceAsCode } from '@lightdash/common';
|
|
2
|
+
import { type SpaceCodeFile } from './spacesAsCode';
|
|
2
3
|
export type DownloadHandlerOptions = {
|
|
3
4
|
verbose: boolean;
|
|
4
5
|
charts: string[];
|
|
@@ -21,7 +22,9 @@ export type DownloadHandlerOptions = {
|
|
|
21
22
|
public: boolean;
|
|
22
23
|
includeCharts: boolean;
|
|
23
24
|
nested: boolean;
|
|
25
|
+
rootSpaces: boolean;
|
|
24
26
|
skipSpaces: boolean;
|
|
27
|
+
spacesOnly?: boolean;
|
|
25
28
|
skipCharts: boolean;
|
|
26
29
|
skipDashboards: boolean;
|
|
27
30
|
skipAlerts: boolean;
|
|
@@ -48,17 +51,27 @@ type MetadataEntry = {
|
|
|
48
51
|
downloadedAt: string;
|
|
49
52
|
};
|
|
50
53
|
type DownloadContentType = 'charts' | 'dashboards' | 'sqlCharts';
|
|
51
|
-
export declare const downloadContent: (ids: string[], type: DownloadContentType, projectId: string, projectName: string, customPath?: string, languageMap?: boolean, nested?: boolean, skipSpaces?: boolean, stripPivotSeries?: boolean) => Promise<[number, string[], MetadataEntry[], SpaceAsCode[]]>;
|
|
54
|
+
export declare const downloadContent: (ids: string[], type: DownloadContentType, projectId: string, projectName: string, customPath?: string, languageMap?: boolean, nested?: boolean, skipSpaces?: boolean, stripPivotSeries?: boolean, rootSpaces?: boolean) => Promise<[number, string[], MetadataEntry[], SpaceAsCode[]]>;
|
|
52
55
|
export declare const downloadHandler: (options: DownloadHandlerOptions) => Promise<void>;
|
|
53
56
|
export declare const uploadHandler: (options: DownloadHandlerOptions) => Promise<void>;
|
|
54
57
|
export declare const testHelpers: {
|
|
58
|
+
assertUniqueSpacePaths: (files: SpaceCodeFile[]) => void;
|
|
59
|
+
downloadSpaces: (projectId: string, projectName: string, customPath?: string, nested?: boolean, rootSpaces?: boolean) => Promise<number>;
|
|
60
|
+
getFlatSpaceFileNames: (spaces: SpaceAsCode[]) => string[];
|
|
55
61
|
getDashboardChartSlugs: (dashboardSlugs: string[], customPath?: string, looseDashboards?: (DashboardAsCode & {
|
|
56
62
|
needsUpdating: boolean;
|
|
57
63
|
})[]) => Promise<string[]>;
|
|
58
64
|
readAiAgentFiles: (customPath?: string) => Promise<AgentAsCode[]>;
|
|
65
|
+
readSpaceFiles: (customPath?: string) => Promise<SpaceCodeFile[]>;
|
|
66
|
+
readSpaceNames: (customPath?: string) => Promise<Record<string, string>>;
|
|
59
67
|
sanitizeChartForDownload: (chart: ChartAsCode, stripPivotSeries: boolean) => ChartAsCode;
|
|
68
|
+
shouldFallBackToEmbeddedSpaces: (error: unknown, spacesOnly: boolean | undefined) => boolean;
|
|
60
69
|
shouldDownloadAiAgents: ({ includeAll, includeAgents, agents, appsOnly, }: Pick<DownloadHandlerOptions, "includeAll" | "includeAgents" | "agents" | "appsOnly">) => boolean;
|
|
70
|
+
sortSpaceFilesParentFirst: (files: SpaceCodeFile[]) => SpaceCodeFile[];
|
|
71
|
+
upsertSpaces: (projectId: string, files: SpaceCodeFile[], changes: Record<string, number>, skipSpaceCreate: boolean, publicSpaceCreate: boolean) => Promise<Record<string, number>>;
|
|
61
72
|
upsertVirtualViews: (projectId: string, slugs: string[], changes: Record<string, number>, force: boolean, canUpload: boolean, customPath?: string) => Promise<Record<string, number>>;
|
|
73
|
+
validateSpaceIdentity: (parsed: unknown, filePath: string) => SpaceAsCode;
|
|
74
|
+
writeSpaceFiles: (spaces: SpaceAsCode[], projectName: string, customPath?: string, folderScheme?: "flat" | "nested", flatSpaceLayout?: "folder" | "root", preserveExistingPolicy?: boolean) => Promise<void>;
|
|
62
75
|
};
|
|
63
76
|
export {};
|
|
64
77
|
//# sourceMappingURL=download.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../src/handlers/download.ts"],"names":[],"mappings":"AAEA,OAAO,EACH,WAAW,EAuBX,WAAW,EAGX,eAAe,EAef,KAAK,WAAW,EACnB,MAAM,mBAAmB,CAAC;
|
|
1
|
+
{"version":3,"file":"download.d.ts","sourceRoot":"","sources":["../../src/handlers/download.ts"],"names":[],"mappings":"AAEA,OAAO,EACH,WAAW,EAuBX,WAAW,EAGX,eAAe,EAef,KAAK,WAAW,EACnB,MAAM,mBAAmB,CAAC;AA4D3B,OAAO,EAmBH,KAAK,aAAa,EACrB,MAAM,gBAAgB,CAAC;AAExB,MAAM,MAAM,sBAAsB,GAAG;IACjC,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,mBAAmB,EAAE,MAAM,EAAE,CAAC;IAC9B,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,KAAK,EAAE,OAAO,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,OAAO,CAAC;IACrB,eAAe,EAAE,OAAO,CAAC;IACzB,MAAM,EAAE,OAAO,CAAC;IAChB,aAAa,EAAE,OAAO,CAAC;IACvB,MAAM,EAAE,OAAO,CAAC;IAChB,UAAU,EAAE,OAAO,CAAC;IACpB,UAAU,EAAE,OAAO,CAAC;IACpB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,UAAU,EAAE,OAAO,CAAC;IACpB,cAAc,EAAE,OAAO,CAAC;IACxB,UAAU,EAAE,OAAO,CAAC;IACpB,UAAU,EAAE,OAAO,CAAC;IACpB,gBAAgB,EAAE,OAAO,CAAC;IAC1B,uBAAuB,EAAE,OAAO,CAAC;IACjC,gBAAgB,EAAE,OAAO,CAAC;IAC1B,aAAa,EAAE,OAAO,CAAC;IACvB,mBAAmB,EAAE,OAAO,CAAC;IAC7B,0BAA0B,EAAE,OAAO,CAAC;IACpC,mBAAmB,EAAE,OAAO,CAAC;IAC7B,UAAU,EAAE,OAAO,CAAC;IACpB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,gBAAgB,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,YAAY,EAAE,OAAO,CAAC;IACtB,WAAW,CAAC,EAAE,OAAO,CAAC;CACzB,CAAC;AA8FF,KAAK,aAAa,GAAG;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,QAAQ,GAAG,YAAY,CAAC;IAC9B,YAAY,EAAE,MAAM,CAAC;CACxB,CAAC;AA6XF,KAAK,mBAAmB,GAAG,QAAQ,GAAG,YAAY,GAAG,WAAW,CAAC;AAkDjE,eAAO,MAAM,eAAe,GACxB,KAAK,MAAM,EAAE,EACb,MAAM,mBAAmB,EACzB,WAAW,MAAM,EACjB,aAAa,MAAM,EACnB,aAAa,MAAM,EACnB,cAAa,OAAe,EAC5B,SAAQ,OAAe,EACvB,aAAY,OAAe,EAC3B,mBAAkB,OAAe,EACjC,aAAY,OAAe,KAC5B,OAAO,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,aAAa,EAAE,EAAE,WAAW,EAAE,CAAC,CAuI5D,CAAC;AA4gBF,eAAO,MAAM,eAAe,GACxB,SAAS,sBAAsB,KAChC,OAAO,CAAC,IAAI,CAglBd,CAAC;AAgaF,eAAO,MAAM,aAAa,GACtB,SAAS,sBAAsB,KAChC,OAAO,CAAC,IAAI,CA6pBd,CAAC;AAEF,eAAO,MAAM,WAAW;;;;6CAlsBJ,MAAM,EAAE,eACX,MAAM,oBACF,CAAC,eAAe,GAAG;QAAE,aAAa,EAAE,OAAO,CAAA;KAAE,CAAC,EAAE;oCAruCpD,MAAM,KACpB,OAAO,CAAC,WAAW,EAAE,CAAC;;;sCAzzBd,WAAW,oBACA,OAAO,KAC1B,WAAW;;+EA9FX,IAAI,CACH,sBAAsB,EACtB,YAAY,GAAG,eAAe,GAAG,QAAQ,GAAG,UAAU,CACzD,KAAG,OAAO;;;oCAmvBI,MAAM,SACV,MAAM,EAAE,WACN,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,SACxB,OAAO,aACH,OAAO,eACL,MAAM,KACpB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;;;CA6kEjC,CAAC"}
|
|
@@ -24,6 +24,7 @@ const apiClient_1 = require("./dbt/apiClient");
|
|
|
24
24
|
const metadataFile_1 = require("./metadataFile");
|
|
25
25
|
const organizationContent_1 = require("./organizationContent");
|
|
26
26
|
const selectProject_1 = require("./selectProject");
|
|
27
|
+
const spacesAsCode_1 = require("./spacesAsCode");
|
|
27
28
|
const shouldDownloadAiAgents = ({ includeAll, includeAgents, agents, appsOnly, }) => appsOnly !== true &&
|
|
28
29
|
(includeAll === true || includeAgents === true || agents.length > 0);
|
|
29
30
|
/*
|
|
@@ -110,76 +111,21 @@ const writeContent = async (contentAsCode, outputDir, languageMap, stripPivotSer
|
|
|
110
111
|
downloadedAt: downloadedAtString,
|
|
111
112
|
};
|
|
112
113
|
};
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
if (!seen.has(space.slug)) {
|
|
126
|
-
seen.add(space.slug);
|
|
127
|
-
let outputDir;
|
|
128
|
-
if (folderScheme === 'nested') {
|
|
129
|
-
outputDir = path.join(baseDir, projectName, space.slug);
|
|
130
|
-
}
|
|
131
|
-
else {
|
|
132
|
-
outputDir = baseDir;
|
|
133
|
-
}
|
|
134
|
-
await fs_1.promises.mkdir(outputDir, { recursive: true });
|
|
135
|
-
const fileName = `${(0, common_1.generateSlug)(space.spaceName)}.space.yml`;
|
|
136
|
-
const filePath = path.join(outputDir, fileName);
|
|
137
|
-
const content = yaml.dump(space, {
|
|
138
|
-
quotingType: '"',
|
|
139
|
-
sortKeys: true,
|
|
140
|
-
});
|
|
141
|
-
await fs_1.promises.writeFile(filePath, content);
|
|
142
|
-
globalState_1.default.debug(`Wrote space file: ${filePath}`);
|
|
143
|
-
}
|
|
144
|
-
}
|
|
145
|
-
};
|
|
146
|
-
/**
|
|
147
|
-
* Reads all .space.yml files from the download directory and returns
|
|
148
|
-
* a map of space slug → original space name. Used during upload to
|
|
149
|
-
* preserve human-readable space names instead of deriving them from slugs.
|
|
150
|
-
*/
|
|
151
|
-
const readSpaceNames = async (customPath) => {
|
|
152
|
-
const baseDir = (0, contentAsCodePaths_1.getDownloadFolder)(customPath);
|
|
153
|
-
const spaceNames = {};
|
|
154
|
-
try {
|
|
155
|
-
const allEntries = await fs_1.promises.readdir(baseDir, {
|
|
156
|
-
recursive: true,
|
|
157
|
-
withFileTypes: true,
|
|
158
|
-
});
|
|
159
|
-
await Promise.all(allEntries
|
|
160
|
-
.filter((entry) => entry.isFile() && entry.name.endsWith('.space.yml'))
|
|
161
|
-
.map(async (file) => {
|
|
162
|
-
try {
|
|
163
|
-
const filePath = path.join(file.parentPath, file.name);
|
|
164
|
-
const fileContent = await fs_1.promises.readFile(filePath, 'utf-8');
|
|
165
|
-
const parsed = yaml.load(fileContent);
|
|
166
|
-
if (parsed?.contentType ===
|
|
167
|
-
common_1.ContentAsCodeType.SPACE &&
|
|
168
|
-
typeof parsed.slug === 'string' &&
|
|
169
|
-
typeof parsed.spaceName === 'string') {
|
|
170
|
-
spaceNames[parsed.slug] = parsed.spaceName;
|
|
171
|
-
}
|
|
172
|
-
}
|
|
173
|
-
catch (e) {
|
|
174
|
-
globalState_1.default.debug(`Skipping space file ${file.name}: ${(0, common_1.getErrorMessage)(e)}`);
|
|
175
|
-
}
|
|
176
|
-
}));
|
|
177
|
-
}
|
|
178
|
-
catch {
|
|
179
|
-
// Directory doesn't exist or can't be read — return empty map
|
|
114
|
+
function getPromoteAction(action) {
|
|
115
|
+
switch (action) {
|
|
116
|
+
case common_1.PromotionAction.CREATE:
|
|
117
|
+
return 'created';
|
|
118
|
+
case common_1.PromotionAction.UPDATE:
|
|
119
|
+
return 'updated';
|
|
120
|
+
case common_1.PromotionAction.DELETE:
|
|
121
|
+
return 'deleted';
|
|
122
|
+
case common_1.PromotionAction.NO_CHANGES:
|
|
123
|
+
return 'skipped';
|
|
124
|
+
default:
|
|
125
|
+
(0, common_1.assertUnreachable)(action, `Unknown promotion action: ${action}`);
|
|
180
126
|
}
|
|
181
|
-
return
|
|
182
|
-
}
|
|
127
|
+
return 'skipped';
|
|
128
|
+
}
|
|
183
129
|
const hasUnsortedKeys = (obj) => {
|
|
184
130
|
if (typeof obj !== 'object' || obj === null || Array.isArray(obj)) {
|
|
185
131
|
if (Array.isArray(obj)) {
|
|
@@ -198,6 +144,7 @@ const isLightdashContentFile = (folder, entry) => entry.isFile() &&
|
|
|
198
144
|
entry.parentPath &&
|
|
199
145
|
entry.parentPath.endsWith(path.sep + folder) &&
|
|
200
146
|
entry.name.endsWith('.yml') &&
|
|
147
|
+
!entry.name.endsWith('.space.yml') &&
|
|
201
148
|
!entry.name.endsWith('.language.map.yml');
|
|
202
149
|
const isLooseContentFile = (entry) => entry.isFile() &&
|
|
203
150
|
entry.parentPath &&
|
|
@@ -297,7 +244,7 @@ const readLooseCodeFiles = async (customPath) => {
|
|
|
297
244
|
dashboards.push(processYamlItem(parsed, file.name, stats, 'dashboards', metadata));
|
|
298
245
|
}
|
|
299
246
|
else if (contentType === common_1.ContentAsCodeType.SPACE) {
|
|
300
|
-
// Space
|
|
247
|
+
// Space files are handled by the dedicated space phase.
|
|
301
248
|
}
|
|
302
249
|
else {
|
|
303
250
|
globalState_1.default.debug(`Skipping ${file.name}: no recognized contentType`);
|
|
@@ -369,7 +316,7 @@ const extractChartSlugsFromDashboards = (dashboards) => dashboards.reduce((acc,
|
|
|
369
316
|
.filter((slug) => slug !== undefined);
|
|
370
317
|
return [...acc, ...slugs];
|
|
371
318
|
}, []);
|
|
372
|
-
const downloadContent = async (ids, type, projectId, projectName, customPath, languageMap = false, nested = false, skipSpaces = false, stripPivotSeries = false) => {
|
|
319
|
+
const downloadContent = async (ids, type, projectId, projectName, customPath, languageMap = false, nested = false, skipSpaces = false, stripPivotSeries = false, rootSpaces = false) => {
|
|
373
320
|
const spinner = globalState_1.default.getActiveSpinner();
|
|
374
321
|
const contentFilters = parseContentFilters(ids);
|
|
375
322
|
const folderScheme = nested ? 'nested' : 'flat';
|
|
@@ -466,7 +413,10 @@ const downloadContent = async (ids, type, projectId, projectName, customPath, la
|
|
|
466
413
|
} while (offset < total);
|
|
467
414
|
// Write space YAML files
|
|
468
415
|
if (!skipSpaces) {
|
|
469
|
-
|
|
416
|
+
const uniqueSpaces = [
|
|
417
|
+
...new Map(allSpaces.map((space) => [space.slug, space])).values(),
|
|
418
|
+
];
|
|
419
|
+
await (0, spacesAsCode_1.writeSpaceFiles)(uniqueSpaces, projectName, customPath, folderScheme, rootSpaces ? 'root' : 'folder', true);
|
|
470
420
|
}
|
|
471
421
|
return [total, [...new Set(chartSlugs)], allMetadataEntries, allSpaces];
|
|
472
422
|
};
|
|
@@ -476,21 +426,6 @@ const getAiAgentsFolder = (customPath) => path.join((0, contentAsCodePaths_1.get
|
|
|
476
426
|
const getAlertsFolder = (customPath) => path.join((0, contentAsCodePaths_1.getDownloadFolder)(customPath), 'alerts');
|
|
477
427
|
const getGoogleSheetsFolder = (customPath) => path.join((0, contentAsCodePaths_1.getDownloadFolder)(customPath), 'google-sheets');
|
|
478
428
|
const getVirtualViewsFolder = (customPath) => path.join((0, contentAsCodePaths_1.getDownloadFolder)(customPath), 'virtual-views');
|
|
479
|
-
function getPromoteAction(action) {
|
|
480
|
-
switch (action) {
|
|
481
|
-
case common_1.PromotionAction.CREATE:
|
|
482
|
-
return 'created';
|
|
483
|
-
case common_1.PromotionAction.UPDATE:
|
|
484
|
-
return 'updated';
|
|
485
|
-
case common_1.PromotionAction.DELETE:
|
|
486
|
-
return 'deleted';
|
|
487
|
-
case common_1.PromotionAction.NO_CHANGES:
|
|
488
|
-
return 'skipped';
|
|
489
|
-
default:
|
|
490
|
-
(0, common_1.assertUnreachable)(action, `Unknown promotion action: ${action}`);
|
|
491
|
-
}
|
|
492
|
-
return 'skipped';
|
|
493
|
-
}
|
|
494
429
|
const downloadVirtualViews = async (projectId, slugs, customPath) => {
|
|
495
430
|
const folder = getVirtualViewsFolder(customPath);
|
|
496
431
|
await fs_1.promises.mkdir(folder, { recursive: true });
|
|
@@ -813,8 +748,8 @@ const downloadHandler = async (options) => {
|
|
|
813
748
|
globalState_1.default.setVerbose(options.verbose);
|
|
814
749
|
const isOrganizationDownload = options.organization === true;
|
|
815
750
|
const includeAll = options.includeAll === true;
|
|
816
|
-
const includeApps = options.includeApps === true || includeAll;
|
|
817
|
-
const includeAllOptionalContent = includeAll && !options.appsOnly;
|
|
751
|
+
const includeApps = !options.spacesOnly && (options.includeApps === true || includeAll);
|
|
752
|
+
const includeAllOptionalContent = includeAll && !options.appsOnly && !options.spacesOnly;
|
|
818
753
|
const { limit: appsLimit, noEffectWarning: appsLimitWarning } = (0, appsDownload_1.resolveAppsLimit)(options.appsLimit, includeApps);
|
|
819
754
|
if (appsLimitWarning) {
|
|
820
755
|
globalState_1.default.log(styles.warning(appsLimitWarning));
|
|
@@ -836,6 +771,46 @@ const downloadHandler = async (options) => {
|
|
|
836
771
|
options.includeScheduledDeliveries = false;
|
|
837
772
|
options.includeVirtualViews = false;
|
|
838
773
|
}
|
|
774
|
+
if (options.spacesOnly) {
|
|
775
|
+
if (options.skipSpaces) {
|
|
776
|
+
throw new common_1.ParameterError('Nothing to download: --spaces-only cannot be combined with --skip-spaces.');
|
|
777
|
+
}
|
|
778
|
+
options.skipCharts = true;
|
|
779
|
+
options.skipDashboards = true;
|
|
780
|
+
options.agents = [];
|
|
781
|
+
options.alerts = [];
|
|
782
|
+
options.apps = [];
|
|
783
|
+
options.googleSheets = [];
|
|
784
|
+
options.scheduledDeliveries = [];
|
|
785
|
+
options.virtualViews = [];
|
|
786
|
+
options.includeAgents = false;
|
|
787
|
+
options.includeApps = false;
|
|
788
|
+
options.includeAlerts = false;
|
|
789
|
+
options.includeGoogleSheets = false;
|
|
790
|
+
options.includeScheduledDeliveries = false;
|
|
791
|
+
options.includeVirtualViews = false;
|
|
792
|
+
}
|
|
793
|
+
if (options.rootSpaces && options.nested) {
|
|
794
|
+
throw new common_1.ParameterError('--root-spaces cannot be combined with --nested');
|
|
795
|
+
}
|
|
796
|
+
const hasFilters = !options.spacesOnly &&
|
|
797
|
+
(options.charts.length > 0 ||
|
|
798
|
+
options.dashboards.length > 0 ||
|
|
799
|
+
options.agents.length > 0 ||
|
|
800
|
+
options.alerts.length > 0 ||
|
|
801
|
+
options.googleSheets.length > 0 ||
|
|
802
|
+
options.scheduledDeliveries.length > 0 ||
|
|
803
|
+
options.virtualViews.length > 0);
|
|
804
|
+
const shouldDownloadSpaces = !isOrganizationDownload && !options.skipSpaces && !hasFilters;
|
|
805
|
+
let skipEmbeddedSpaces = !hasFilters || options.skipSpaces;
|
|
806
|
+
if (shouldDownloadSpaces) {
|
|
807
|
+
try {
|
|
808
|
+
await (0, spacesAsCode_1.getUniqueExistingSpaceFilePathsBySlug)((0, contentAsCodePaths_1.getDownloadFolder)(options.path));
|
|
809
|
+
}
|
|
810
|
+
catch (error) {
|
|
811
|
+
throw (0, spacesAsCode_1.createSpaceAsCodeDownloadError)((0, common_1.getErrorMessage)(error));
|
|
812
|
+
}
|
|
813
|
+
}
|
|
839
814
|
await (0, apiClient_1.checkLightdashVersion)();
|
|
840
815
|
const config = await (0, config_1.getConfig)();
|
|
841
816
|
if (!config.context?.apiKey || !config.context.serverUrl) {
|
|
@@ -860,8 +835,9 @@ const downloadHandler = async (options) => {
|
|
|
860
835
|
const projectId = projectSelection.projectUuid;
|
|
861
836
|
// Log current project info
|
|
862
837
|
(0, selectProject_1.logSelectedProject)(projectSelection, config, 'Downloading from');
|
|
863
|
-
const spinner = globalState_1.default.startSpinner(
|
|
864
|
-
|
|
838
|
+
const spinner = globalState_1.default.startSpinner(options.spacesOnly
|
|
839
|
+
? `Downloading spaces`
|
|
840
|
+
: `Downloading content from project`);
|
|
865
841
|
// Fetch project details to get project name for folder structure
|
|
866
842
|
const project = await (0, apiClient_1.lightdashApi)({
|
|
867
843
|
method: 'GET',
|
|
@@ -882,17 +858,25 @@ const downloadHandler = async (options) => {
|
|
|
882
858
|
},
|
|
883
859
|
});
|
|
884
860
|
try {
|
|
885
|
-
const hasFilters = options.charts.length > 0 ||
|
|
886
|
-
options.dashboards.length > 0 ||
|
|
887
|
-
options.agents.length > 0 ||
|
|
888
|
-
options.alerts.length > 0 ||
|
|
889
|
-
options.googleSheets.length > 0 ||
|
|
890
|
-
options.scheduledDeliveries.length > 0 ||
|
|
891
|
-
options.virtualViews.length > 0;
|
|
892
|
-
// When downloading specific charts or dashboards, skip space metadata
|
|
893
|
-
const skipSpaces = options.skipSpaces || hasFilters;
|
|
894
861
|
let allMetadataEntries = [];
|
|
895
|
-
|
|
862
|
+
if (shouldDownloadSpaces) {
|
|
863
|
+
if (!options.spacesOnly)
|
|
864
|
+
spinner.start(`Downloading spaces`);
|
|
865
|
+
try {
|
|
866
|
+
const spaceTotal = await (0, spacesAsCode_1.downloadSpaces)(projectId, projectName, options.path, options.nested, options.rootSpaces);
|
|
867
|
+
spinner.succeed(`Downloaded ${spaceTotal} spaces`);
|
|
868
|
+
}
|
|
869
|
+
catch (error) {
|
|
870
|
+
if (!(0, spacesAsCode_1.shouldFallBackToEmbeddedSpaces)(error, options.spacesOnly)) {
|
|
871
|
+
throw (0, spacesAsCode_1.isSpaceAsCodeFetchError)(error)
|
|
872
|
+
? (0, spacesAsCode_1.createSpaceAsCodeDownloadError)((0, common_1.getErrorMessage)(error))
|
|
873
|
+
: error;
|
|
874
|
+
}
|
|
875
|
+
skipEmbeddedSpaces = false;
|
|
876
|
+
spinner.warn(styles.warning('Space access is unavailable; continuing with legacy space metadata where available.'));
|
|
877
|
+
globalState_1.default.debug(`Could not download access-aware spaces: ${(0, common_1.getErrorMessage)(error)}`);
|
|
878
|
+
}
|
|
879
|
+
}
|
|
896
880
|
if (includeAllOptionalContent ||
|
|
897
881
|
options.includeVirtualViews ||
|
|
898
882
|
options.virtualViews.length > 0) {
|
|
@@ -906,19 +890,17 @@ const downloadHandler = async (options) => {
|
|
|
906
890
|
globalState_1.default.log(styles.warning(`No charts filters provided, skipping`));
|
|
907
891
|
}
|
|
908
892
|
else {
|
|
909
|
-
const [regularChartTotal, , regularChartMeta
|
|
893
|
+
const [regularChartTotal, , regularChartMeta] = await (0, exports.downloadContent)(options.charts, 'charts', projectId, projectName, options.path, options.languageMap, options.nested, skipEmbeddedSpaces, options.stripPivotSeries, options.rootSpaces);
|
|
910
894
|
spinner.succeed(`Downloaded ${regularChartTotal} charts`);
|
|
911
895
|
allMetadataEntries = [
|
|
912
896
|
...allMetadataEntries,
|
|
913
897
|
...regularChartMeta,
|
|
914
898
|
];
|
|
915
|
-
allSpaces = [...allSpaces, ...regularChartSpaces];
|
|
916
899
|
// Download SQL charts
|
|
917
900
|
spinner.start(`Downloading SQL charts`);
|
|
918
|
-
const [sqlChartTotal, , sqlChartMeta
|
|
901
|
+
const [sqlChartTotal, , sqlChartMeta] = await (0, exports.downloadContent)(options.charts, 'sqlCharts', projectId, projectName, options.path, options.languageMap, options.nested, skipEmbeddedSpaces, false, options.rootSpaces);
|
|
919
902
|
spinner.succeed(`Downloaded ${sqlChartTotal} SQL charts`);
|
|
920
903
|
allMetadataEntries = [...allMetadataEntries, ...sqlChartMeta];
|
|
921
|
-
allSpaces = [...allSpaces, ...sqlChartSpaces];
|
|
922
904
|
chartTotal = regularChartTotal + sqlChartTotal;
|
|
923
905
|
}
|
|
924
906
|
}
|
|
@@ -930,33 +912,28 @@ const downloadHandler = async (options) => {
|
|
|
930
912
|
else {
|
|
931
913
|
let chartSlugs = [];
|
|
932
914
|
let dashMeta;
|
|
933
|
-
|
|
934
|
-
[dashboardTotal, chartSlugs, dashMeta, dashSpaces] =
|
|
935
|
-
await (0, exports.downloadContent)(options.dashboards, 'dashboards', projectId, projectName, options.path, options.languageMap, options.nested, skipSpaces, false);
|
|
915
|
+
[dashboardTotal, chartSlugs, dashMeta] = await (0, exports.downloadContent)(options.dashboards, 'dashboards', projectId, projectName, options.path, options.languageMap, options.nested, skipEmbeddedSpaces, false, options.rootSpaces);
|
|
936
916
|
allMetadataEntries = [...allMetadataEntries, ...dashMeta];
|
|
937
|
-
allSpaces = [...allSpaces, ...dashSpaces];
|
|
938
917
|
spinner.succeed(`Downloaded ${dashboardTotal} dashboards`);
|
|
939
918
|
if (hasFilters &&
|
|
940
919
|
chartSlugs.length > 0 &&
|
|
941
920
|
!options.skipCharts) {
|
|
942
921
|
spinner.start(`Downloading ${chartSlugs.length} charts linked to dashboards`);
|
|
943
|
-
const [regularCharts, , linkedChartMeta
|
|
922
|
+
const [regularCharts, , linkedChartMeta] = await (0, exports.downloadContent)(chartSlugs, 'charts', projectId, projectName, options.path, options.languageMap, options.nested, skipEmbeddedSpaces, options.stripPivotSeries, options.rootSpaces);
|
|
944
923
|
allMetadataEntries = [
|
|
945
924
|
...allMetadataEntries,
|
|
946
925
|
...linkedChartMeta,
|
|
947
926
|
];
|
|
948
|
-
|
|
949
|
-
const [sqlCharts, , linkedSqlMeta, linkedSqlSpaces] = await (0, exports.downloadContent)(chartSlugs, 'sqlCharts', projectId, projectName, options.path, options.languageMap, options.nested, skipSpaces);
|
|
927
|
+
const [sqlCharts, , linkedSqlMeta] = await (0, exports.downloadContent)(chartSlugs, 'sqlCharts', projectId, projectName, options.path, options.languageMap, options.nested, skipEmbeddedSpaces, false, options.rootSpaces);
|
|
950
928
|
allMetadataEntries = [
|
|
951
929
|
...allMetadataEntries,
|
|
952
930
|
...linkedSqlMeta,
|
|
953
931
|
];
|
|
954
|
-
allSpaces = [...allSpaces, ...linkedSqlSpaces];
|
|
955
932
|
spinner.succeed(`Downloaded ${regularCharts + sqlCharts} charts linked to dashboards`);
|
|
956
933
|
}
|
|
957
934
|
}
|
|
958
935
|
}
|
|
959
|
-
if (shouldDownloadAiAgents(options)) {
|
|
936
|
+
if (!options.spacesOnly && shouldDownloadAiAgents(options)) {
|
|
960
937
|
spinner.start(`Downloading AI agents`);
|
|
961
938
|
const agentTotal = await downloadAiAgents(projectId, options.agents, options.path);
|
|
962
939
|
spinner.succeed(`Downloaded ${agentTotal} AI agents`);
|
|
@@ -1101,11 +1078,6 @@ const downloadHandler = async (options) => {
|
|
|
1101
1078
|
}
|
|
1102
1079
|
}
|
|
1103
1080
|
}
|
|
1104
|
-
// Report space definitions count
|
|
1105
|
-
if (!skipSpaces) {
|
|
1106
|
-
const uniqueSpaceCount = new Set(allSpaces.map((s) => s.slug)).size;
|
|
1107
|
-
spinner.succeed(`Downloaded ${uniqueSpaceCount} space definitions`);
|
|
1108
|
-
}
|
|
1109
1081
|
// Write metadata file with all downloadedAt timestamps
|
|
1110
1082
|
const metadataToWrite = {
|
|
1111
1083
|
version: 1,
|
|
@@ -1149,6 +1121,8 @@ const downloadHandler = async (options) => {
|
|
|
1149
1121
|
error: `${error}`,
|
|
1150
1122
|
},
|
|
1151
1123
|
});
|
|
1124
|
+
if ((0, spacesAsCode_1.isSpaceAsCodeDownloadError)(error))
|
|
1125
|
+
throw error;
|
|
1152
1126
|
}
|
|
1153
1127
|
};
|
|
1154
1128
|
exports.downloadHandler = downloadHandler;
|
|
@@ -1173,14 +1147,6 @@ const storeUploadChanges = (changes, promoteChanges) => {
|
|
|
1173
1147
|
});
|
|
1174
1148
|
return updatedChanges;
|
|
1175
1149
|
};
|
|
1176
|
-
const logUploadChanges = (changes) => {
|
|
1177
|
-
Object.entries(changes).forEach(([key, value]) => {
|
|
1178
|
-
console.info(`Total ${key}: ${value} `);
|
|
1179
|
-
});
|
|
1180
|
-
if ((0, appsDownload_1.shouldWarnAllSkipped)(changes)) {
|
|
1181
|
-
console.warn(styles.warning(`\nAll content was skipped (no local changes detected). Use --force to upload all content, e.g. when uploading to a new project.`));
|
|
1182
|
-
}
|
|
1183
|
-
};
|
|
1184
1150
|
// SQL charts have 'sql' field instead of 'tableName'/'metricQuery'
|
|
1185
1151
|
const isSqlChart = (item) => 'sql' in item && !('tableName' in item);
|
|
1186
1152
|
const upsertSingleItem = async (item, type, projectId, changes, force, config, skipSpaceCreate, publicSpaceCreate, validate, spaceNames) => {
|
|
@@ -1386,7 +1352,28 @@ const getDashboardChartSlugs = async (dashboardSlugs, customPath, looseDashboard
|
|
|
1386
1352
|
};
|
|
1387
1353
|
const uploadHandler = async (options) => {
|
|
1388
1354
|
globalState_1.default.setVerbose(options.verbose);
|
|
1355
|
+
if (options.spacesOnly && options.skipSpaces) {
|
|
1356
|
+
throw new common_1.ParameterError('Nothing to upload: --spaces-only cannot be combined with --skip-spaces.');
|
|
1357
|
+
}
|
|
1389
1358
|
const isOrganizationUpload = options.organization === true;
|
|
1359
|
+
const hasFilters = !options.spacesOnly &&
|
|
1360
|
+
(options.charts.length > 0 ||
|
|
1361
|
+
options.dashboards.length > 0 ||
|
|
1362
|
+
options.agents.length > 0 ||
|
|
1363
|
+
options.alerts.length > 0 ||
|
|
1364
|
+
options.googleSheets.length > 0 ||
|
|
1365
|
+
options.scheduledDeliveries.length > 0 ||
|
|
1366
|
+
options.virtualViews.length > 0);
|
|
1367
|
+
const shouldReconcileSpaces = !isOrganizationUpload && !options.skipSpaces && !hasFilters;
|
|
1368
|
+
let preflightSpaceFiles = [];
|
|
1369
|
+
if (shouldReconcileSpaces) {
|
|
1370
|
+
try {
|
|
1371
|
+
preflightSpaceFiles = await (0, spacesAsCode_1.readSpaceFiles)(options.path);
|
|
1372
|
+
}
|
|
1373
|
+
catch (error) {
|
|
1374
|
+
throw (0, spacesAsCode_1.createSpaceAsCodeUploadError)((0, common_1.getErrorMessage)(error));
|
|
1375
|
+
}
|
|
1376
|
+
}
|
|
1390
1377
|
if (options.gzip) {
|
|
1391
1378
|
(0, apiClient_1.setGzipEnabled)(true);
|
|
1392
1379
|
}
|
|
@@ -1430,16 +1417,34 @@ const uploadHandler = async (options) => {
|
|
|
1430
1417
|
});
|
|
1431
1418
|
let uploadResource = 'content as code';
|
|
1432
1419
|
try {
|
|
1420
|
+
uploadResource = 'space definitions';
|
|
1421
|
+
const spaceFiles = preflightSpaceFiles;
|
|
1422
|
+
const spaceNames = shouldReconcileSpaces
|
|
1423
|
+
? (0, spacesAsCode_1.getSpaceNames)(spaceFiles)
|
|
1424
|
+
: await (0, spacesAsCode_1.readSpaceNames)(options.path);
|
|
1425
|
+
if (spaceFiles.length > 0) {
|
|
1426
|
+
globalState_1.default.log(`Found ${spaceFiles.length} space definition(s)`);
|
|
1427
|
+
}
|
|
1428
|
+
if (shouldReconcileSpaces) {
|
|
1429
|
+
changes = await (0, spacesAsCode_1.upsertSpaces)(projectId, spaceFiles, changes, options.skipSpaceCreate, options.public);
|
|
1430
|
+
}
|
|
1431
|
+
else if (hasFilters) {
|
|
1432
|
+
globalState_1.default.debug('Skipping space access reconciliation for a filtered content upload');
|
|
1433
|
+
}
|
|
1434
|
+
if (options.spacesOnly) {
|
|
1435
|
+
await analytics_1.LightdashAnalytics.track({
|
|
1436
|
+
event: 'upload.completed',
|
|
1437
|
+
properties: {
|
|
1438
|
+
userId: config.user?.userUuid,
|
|
1439
|
+
organizationId: config.user?.organizationUuid,
|
|
1440
|
+
projectId,
|
|
1441
|
+
timeToCompleted: (Date.now() - start) / 1000,
|
|
1442
|
+
},
|
|
1443
|
+
});
|
|
1444
|
+
(0, spacesAsCode_1.logUploadChanges)(changes);
|
|
1445
|
+
return;
|
|
1446
|
+
}
|
|
1433
1447
|
const uploadPermissions = await (0, apiClient_1.getContentAsCodeUploadPermissions)(projectId);
|
|
1434
|
-
// If any filter is provided, we skip those items without filters
|
|
1435
|
-
// eg: if a --charts filter is provided, we skip dashboards if no --dashboards filter is provided
|
|
1436
|
-
const hasFilters = options.charts.length > 0 ||
|
|
1437
|
-
options.dashboards.length > 0 ||
|
|
1438
|
-
options.agents.length > 0 ||
|
|
1439
|
-
options.alerts.length > 0 ||
|
|
1440
|
-
options.googleSheets.length > 0 ||
|
|
1441
|
-
options.scheduledDeliveries.length > 0 ||
|
|
1442
|
-
options.virtualViews.length > 0;
|
|
1443
1448
|
// Discover loose YAML files (outside charts/ and dashboards/) classified by contentType
|
|
1444
1449
|
uploadResource = 'content files';
|
|
1445
1450
|
const looseFiles = await readLooseCodeFiles(options.path);
|
|
@@ -1461,12 +1466,6 @@ const uploadHandler = async (options) => {
|
|
|
1461
1466
|
if (parseInt(String(options.concurrency), 10) > 1000) {
|
|
1462
1467
|
globalState_1.default.log(styles.warning(`Concurrency limit exceeded. Using maximum of 1000 instead of ${options.concurrency}`));
|
|
1463
1468
|
}
|
|
1464
|
-
// Read space definition files to preserve original space names during upload
|
|
1465
|
-
uploadResource = 'space definitions';
|
|
1466
|
-
const spaceNames = await readSpaceNames(options.path);
|
|
1467
|
-
if (Object.keys(spaceNames).length > 0) {
|
|
1468
|
-
globalState_1.default.log(`Found ${Object.keys(spaceNames).length} space definition(s)`);
|
|
1469
|
-
}
|
|
1470
1469
|
if (!options.skipVirtualViews) {
|
|
1471
1470
|
if (hasFilters && options.virtualViews.length === 0) {
|
|
1472
1471
|
globalState_1.default.log(styles.warning(`No virtual view filters provided, skipping`));
|
|
@@ -1762,7 +1761,7 @@ const uploadHandler = async (options) => {
|
|
|
1762
1761
|
timeToCompleted: (end - start) / 1000, // in seconds
|
|
1763
1762
|
},
|
|
1764
1763
|
});
|
|
1765
|
-
logUploadChanges(changes);
|
|
1764
|
+
(0, spacesAsCode_1.logUploadChanges)(changes);
|
|
1766
1765
|
}
|
|
1767
1766
|
catch (error) {
|
|
1768
1767
|
globalState_1.default.log(styles.error(`\nError uploading ${uploadResource}: ${(0, common_1.getErrorMessage)(error)}`));
|
|
@@ -1775,16 +1774,27 @@ const uploadHandler = async (options) => {
|
|
|
1775
1774
|
error: (0, common_1.getErrorMessage)(error),
|
|
1776
1775
|
},
|
|
1777
1776
|
});
|
|
1778
|
-
if (
|
|
1777
|
+
if ((0, spacesAsCode_1.isSpaceAsCodeUploadError)(error))
|
|
1778
|
+
throw error;
|
|
1779
|
+
if (error instanceof AiAgentAsCodeUploadError)
|
|
1779
1780
|
throw error.originalError;
|
|
1780
|
-
}
|
|
1781
1781
|
}
|
|
1782
1782
|
};
|
|
1783
1783
|
exports.uploadHandler = uploadHandler;
|
|
1784
1784
|
exports.testHelpers = {
|
|
1785
|
+
assertUniqueSpacePaths: spacesAsCode_1.assertUniqueSpacePaths,
|
|
1786
|
+
downloadSpaces: spacesAsCode_1.downloadSpaces,
|
|
1787
|
+
getFlatSpaceFileNames: spacesAsCode_1.getFlatSpaceFileNames,
|
|
1785
1788
|
getDashboardChartSlugs,
|
|
1786
1789
|
readAiAgentFiles,
|
|
1790
|
+
readSpaceFiles: spacesAsCode_1.readSpaceFiles,
|
|
1791
|
+
readSpaceNames: spacesAsCode_1.readSpaceNames,
|
|
1787
1792
|
sanitizeChartForDownload,
|
|
1793
|
+
shouldFallBackToEmbeddedSpaces: spacesAsCode_1.shouldFallBackToEmbeddedSpaces,
|
|
1788
1794
|
shouldDownloadAiAgents,
|
|
1795
|
+
sortSpaceFilesParentFirst: spacesAsCode_1.sortSpaceFilesParentFirst,
|
|
1796
|
+
upsertSpaces: spacesAsCode_1.upsertSpaces,
|
|
1789
1797
|
upsertVirtualViews,
|
|
1798
|
+
validateSpaceIdentity: spacesAsCode_1.validateSpaceIdentity,
|
|
1799
|
+
writeSpaceFiles: spacesAsCode_1.writeSpaceFiles,
|
|
1790
1800
|
};
|