@lightdash/cli 0.1390.1 → 0.1392.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/handlers/download.d.ts +1 -0
- package/dist/handlers/download.js +124 -107
- package/dist/index.js +1 -0
- package/package.json +3 -3
@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.uploadHandler = exports.downloadHandler = void 0;
|
4
4
|
const tslib_1 = require("tslib");
|
5
5
|
/* eslint-disable no-await-in-loop */
|
6
|
+
/* eslint-disable no-param-reassign */
|
6
7
|
const common_1 = require("@lightdash/common");
|
7
8
|
const fs_1 = require("fs");
|
8
9
|
const yaml = tslib_1.__importStar(require("js-yaml"));
|
@@ -11,56 +12,25 @@ const config_1 = require("../config");
|
|
11
12
|
const globalState_1 = tslib_1.__importDefault(require("../globalState"));
|
12
13
|
const apiClient_1 = require("./dbt/apiClient");
|
13
14
|
const DOWNLOAD_FOLDER = 'lightdash';
|
14
|
-
const
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
const
|
22
|
-
|
23
|
-
|
24
|
-
}
|
25
|
-
const chartsAsCode = await (0, apiClient_1.lightdashApi)({
|
26
|
-
method: 'GET',
|
27
|
-
url: `/api/v1/projects/${projectId}/charts/code`,
|
28
|
-
body: undefined,
|
29
|
-
});
|
30
|
-
console.info(`Downloading ${chartsAsCode.length} charts`);
|
31
|
-
const outputDir = path.join(process.cwd(), DOWNLOAD_FOLDER);
|
32
|
-
console.info(`Creating new path for files on ${outputDir} `);
|
33
|
-
try {
|
34
|
-
await fs_1.promises.mkdir(outputDir, { recursive: true });
|
35
|
-
}
|
36
|
-
catch (error) {
|
37
|
-
// Directory already exists
|
38
|
-
}
|
39
|
-
for (const chart of chartsAsCode) {
|
40
|
-
const chartPath = path.join(outputDir, `${chart.slug}.yml`);
|
41
|
-
globalState_1.default.debug(`> Writing chart to ${chartPath}`);
|
42
|
-
const chartYml = yaml.dump(chart, {
|
15
|
+
const dumpIntoFiles = async (folder, items) => {
|
16
|
+
const outputDir = path.join(process.cwd(), DOWNLOAD_FOLDER, folder);
|
17
|
+
console.info(`Writting ${items.length} ${folder} into ${outputDir}`);
|
18
|
+
// Make directory
|
19
|
+
const created = await fs_1.promises.mkdir(outputDir, { recursive: true });
|
20
|
+
if (created)
|
21
|
+
console.info(`Created new folder: ${outputDir} `);
|
22
|
+
for (const item of items) {
|
23
|
+
const itemPath = path.join(outputDir, `${item.slug}.yml`);
|
24
|
+
const chartYml = yaml.dump(item, {
|
43
25
|
quotingType: '"',
|
44
26
|
});
|
45
|
-
await fs_1.promises.writeFile(
|
27
|
+
await fs_1.promises.writeFile(itemPath, chartYml);
|
46
28
|
}
|
47
|
-
// TODO delete files if chart don't exist ?*/
|
48
29
|
};
|
49
|
-
|
50
|
-
const
|
51
|
-
|
52
|
-
|
53
|
-
const config = await (0, config_1.getConfig)();
|
54
|
-
if (!config.context?.apiKey || !config.context.serverUrl) {
|
55
|
-
throw new common_1.AuthorizationError(`Not logged in. Run 'lightdash login --help'`);
|
56
|
-
}
|
57
|
-
const projectId = config.context.project;
|
58
|
-
if (!projectId) {
|
59
|
-
throw new Error('No project selected. Run lightdash config set-project');
|
60
|
-
}
|
61
|
-
const inputDir = path.join(process.cwd(), DOWNLOAD_FOLDER);
|
62
|
-
console.info(`Reading charts from ${inputDir}`);
|
63
|
-
const charts = [];
|
30
|
+
const readCodeFiles = async (folder) => {
|
31
|
+
const inputDir = path.join(process.cwd(), DOWNLOAD_FOLDER, folder);
|
32
|
+
console.info(`Reading ${folder} from ${inputDir}`);
|
33
|
+
const items = [];
|
64
34
|
try {
|
65
35
|
// Read all files from the lightdash directory
|
66
36
|
const files = await fs_1.promises.readdir(inputDir);
|
@@ -69,91 +39,138 @@ const uploadHandler = async (options) => {
|
|
69
39
|
for (const file of jsonFiles) {
|
70
40
|
const filePath = path.join(inputDir, file);
|
71
41
|
const fileContent = await fs_1.promises.readFile(filePath, 'utf-8');
|
72
|
-
const
|
42
|
+
const item = yaml.load(fileContent);
|
73
43
|
const fileUpdatedAt = (await fs_1.promises.stat(filePath)).mtime;
|
74
44
|
// We override the updatedAt to the file's updatedAt
|
75
45
|
// in case there were some changes made locally
|
76
46
|
// do not override if the file was just created
|
77
|
-
const downloadedAt =
|
78
|
-
? new Date(
|
47
|
+
const downloadedAt = item.downloadedAt
|
48
|
+
? new Date(item.downloadedAt)
|
79
49
|
: undefined;
|
80
50
|
const needsUpdating = downloadedAt &&
|
81
51
|
Math.abs(fileUpdatedAt.getTime() - downloadedAt.getTime()) >
|
82
52
|
30000;
|
83
|
-
const
|
84
|
-
...
|
85
|
-
updatedAt: needsUpdating ? fileUpdatedAt :
|
53
|
+
const locallyUpdatedItem = {
|
54
|
+
...item,
|
55
|
+
updatedAt: needsUpdating ? fileUpdatedAt : item.updatedAt,
|
86
56
|
needsUpdating: needsUpdating ?? true, // if downloadAt is not set, we force the update
|
87
57
|
};
|
88
|
-
|
58
|
+
items.push(locallyUpdatedItem);
|
89
59
|
}
|
90
60
|
}
|
91
61
|
catch (error) {
|
92
62
|
if (error.code === 'ENOENT') {
|
93
|
-
|
63
|
+
console.error(`Directory ${inputDir} not found. Run download command first.`);
|
64
|
+
}
|
65
|
+
else {
|
66
|
+
console.error(`Error reading ${inputDir}: ${error}`);
|
94
67
|
}
|
95
68
|
throw error;
|
96
69
|
}
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
70
|
+
return items;
|
71
|
+
};
|
72
|
+
const downloadHandler = async (options) => {
|
73
|
+
globalState_1.default.setVerbose(options.verbose);
|
74
|
+
await (0, apiClient_1.checkLightdashVersion)();
|
75
|
+
const config = await (0, config_1.getConfig)();
|
76
|
+
if (!config.context?.apiKey || !config.context.serverUrl) {
|
77
|
+
throw new common_1.AuthorizationError(`Not logged in. Run 'lightdash login --help'`);
|
78
|
+
}
|
79
|
+
const projectId = config.context.project;
|
80
|
+
if (!projectId) {
|
81
|
+
throw new Error('No project selected. Run lightdash config set-project');
|
82
|
+
}
|
83
|
+
// Download charts
|
84
|
+
globalState_1.default.debug('Downloading charts');
|
85
|
+
const chartsAsCode = await (0, apiClient_1.lightdashApi)({
|
86
|
+
method: 'GET',
|
87
|
+
url: `/api/v1/projects/${projectId}/charts/code`,
|
88
|
+
body: undefined,
|
89
|
+
});
|
90
|
+
await dumpIntoFiles('charts', chartsAsCode);
|
91
|
+
// Download dashboards
|
92
|
+
globalState_1.default.debug('Downloading dashboards');
|
93
|
+
const dashboardsAsCode = await (0, apiClient_1.lightdashApi)({
|
94
|
+
method: 'GET',
|
95
|
+
url: `/api/v1/projects/${projectId}/dashboards/code`,
|
96
|
+
body: undefined,
|
97
|
+
});
|
98
|
+
await dumpIntoFiles('dashboards', dashboardsAsCode);
|
99
|
+
// TODO delete files if chart don't exist ?*/
|
100
|
+
};
|
101
|
+
exports.downloadHandler = downloadHandler;
|
102
|
+
const storeUploadChanges = (changes, promoteChanges) => {
|
103
|
+
const getPromoteChanges = (resource) => {
|
104
|
+
const promotions = promoteChanges[resource];
|
105
|
+
return promotions.reduce((acc, promoteChange) => {
|
106
|
+
const action = promoteChange.action === common_1.PromotionAction.NO_CHANGES
|
107
|
+
? 'skipped'
|
108
|
+
: promoteChange.action;
|
109
|
+
const key = `${resource} ${action}`;
|
110
|
+
acc[key] = (acc[key] ?? 0) + 1;
|
111
|
+
return acc;
|
112
|
+
}, {});
|
113
|
+
};
|
114
|
+
const updatedChanges = {
|
115
|
+
...changes,
|
116
|
+
};
|
117
|
+
['spaces', 'charts', 'dashboards'].forEach((resource) => {
|
118
|
+
const resourceChanges = getPromoteChanges(resource);
|
119
|
+
Object.entries(resourceChanges).forEach(([key, value]) => {
|
120
|
+
updatedChanges[key] = (updatedChanges[key] ?? 0) + value;
|
121
|
+
});
|
122
|
+
});
|
123
|
+
return updatedChanges;
|
124
|
+
};
|
125
|
+
const logUploadChanges = (changes) => {
|
126
|
+
Object.entries(changes).forEach(([key, value]) => {
|
127
|
+
console.info(`Total ${key}: ${value} `);
|
128
|
+
});
|
129
|
+
};
|
130
|
+
const upsertResources = async (type, projectId, changes, force) => {
|
131
|
+
const items = await readCodeFiles(type);
|
132
|
+
console.info(`Found ${items.length} ${type} files`);
|
133
|
+
for (const item of items) {
|
134
|
+
// If a chart fails to update, we keep updating the rest
|
135
|
+
try {
|
136
|
+
if (!force && !item.needsUpdating) {
|
137
|
+
globalState_1.default.debug(`Skipping ${type} "${item.slug}" with no local changes`);
|
138
|
+
changes[`${type} skipped`] =
|
139
|
+
(changes[`${type} skipped`] ?? 0) + 1;
|
109
140
|
// eslint-disable-next-line no-continue
|
110
141
|
continue;
|
111
142
|
}
|
112
|
-
globalState_1.default.debug(`Upserting
|
113
|
-
const
|
143
|
+
globalState_1.default.debug(`Upserting ${type} ${item.slug}`);
|
144
|
+
const upsertData = await (0, apiClient_1.lightdashApi)({
|
114
145
|
method: 'POST',
|
115
|
-
url: `/api/v1/projects/${projectId}
|
116
|
-
body: JSON.stringify(
|
146
|
+
url: `/api/v1/projects/${projectId}/${type}/${item.slug}/code`,
|
147
|
+
body: JSON.stringify(item),
|
117
148
|
});
|
118
|
-
globalState_1.default.debug(
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
break;
|
126
|
-
default:
|
127
|
-
// ignore the rest
|
128
|
-
}
|
129
|
-
switch (chartData.charts[0].action) {
|
130
|
-
case common_1.PromotionAction.CREATE:
|
131
|
-
created += 1;
|
132
|
-
break;
|
133
|
-
case common_1.PromotionAction.UPDATE:
|
134
|
-
updated += 1;
|
135
|
-
break;
|
136
|
-
case common_1.PromotionAction.DELETE:
|
137
|
-
deleted += 1;
|
138
|
-
break;
|
139
|
-
case common_1.PromotionAction.NO_CHANGES:
|
140
|
-
skipped += 1;
|
141
|
-
break;
|
142
|
-
default:
|
143
|
-
globalState_1.default.debug(`Unknown action: ${chartData.charts[0].action}`);
|
144
|
-
break;
|
145
|
-
}
|
149
|
+
globalState_1.default.debug(`${type} "${item.name}": ${upsertData[type]?.[0].action}`);
|
150
|
+
changes = storeUploadChanges(changes, upsertData);
|
151
|
+
}
|
152
|
+
catch (error) {
|
153
|
+
changes[`${type} with errors`] =
|
154
|
+
(changes[`${type} with errors`] ?? 0) + 1;
|
155
|
+
console.error(`Error upserting ${type}`, error);
|
146
156
|
}
|
147
157
|
}
|
148
|
-
|
149
|
-
|
158
|
+
return changes;
|
159
|
+
};
|
160
|
+
const uploadHandler = async (options) => {
|
161
|
+
globalState_1.default.setVerbose(options.verbose);
|
162
|
+
await (0, apiClient_1.checkLightdashVersion)();
|
163
|
+
const config = await (0, config_1.getConfig)();
|
164
|
+
if (!config.context?.apiKey || !config.context.serverUrl) {
|
165
|
+
throw new common_1.AuthorizationError(`Not logged in. Run 'lightdash login --help'`);
|
166
|
+
}
|
167
|
+
const projectId = config.context.project;
|
168
|
+
if (!projectId) {
|
169
|
+
throw new Error('No project selected. Run lightdash config set-project');
|
150
170
|
}
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
console.info(`Total charts deleted: ${deleted} `); // We should not delete charts from the CLI
|
156
|
-
console.info(`Total spaces created: ${spacesCreated} `);
|
157
|
-
console.info(`Total spaces updated: ${spacesUpdated} `);
|
171
|
+
let changes = {};
|
172
|
+
changes = await upsertResources('charts', projectId, changes, options.force);
|
173
|
+
changes = await upsertResources('dashboards', projectId, changes, options.force);
|
174
|
+
logUploadChanges(changes);
|
158
175
|
};
|
159
176
|
exports.uploadHandler = uploadHandler;
|
package/dist/index.js
CHANGED
@@ -199,6 +199,7 @@ commander_1.program
|
|
199
199
|
.command('upload')
|
200
200
|
.description('Uploads charts and dashboards as code')
|
201
201
|
.option('--verbose', undefined, false)
|
202
|
+
.option('--force', 'Force upload even if local files have not changed, use this when you want to upload files to a new project', false)
|
202
203
|
.action(download_1.uploadHandler);
|
203
204
|
commander_1.program
|
204
205
|
.command('deploy')
|
package/package.json
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
{
|
2
2
|
"name": "@lightdash/cli",
|
3
|
-
"version": "0.
|
3
|
+
"version": "0.1392.0",
|
4
4
|
"license": "MIT",
|
5
5
|
"bin": {
|
6
6
|
"lightdash": "dist/index.js"
|
@@ -11,8 +11,8 @@
|
|
11
11
|
],
|
12
12
|
"dependencies": {
|
13
13
|
"@actions/core": "^1.11.1",
|
14
|
-
"@lightdash/common": "^0.
|
15
|
-
"@lightdash/warehouses": "^0.
|
14
|
+
"@lightdash/common": "^0.1392.0",
|
15
|
+
"@lightdash/warehouses": "^0.1392.0",
|
16
16
|
"@types/columnify": "^1.5.1",
|
17
17
|
"ajv": "^8.11.0",
|
18
18
|
"ajv-formats": "^2.1.1",
|