@lightdash/cli 0.1390.1 → 0.1392.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,5 +1,6 @@
1
1
  export type DownloadHandlerOptions = {
2
2
  verbose: boolean;
3
+ force: boolean;
3
4
  };
4
5
  export declare const downloadHandler: (options: DownloadHandlerOptions) => Promise<void>;
5
6
  export declare const uploadHandler: (options: DownloadHandlerOptions) => Promise<void>;
@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.uploadHandler = exports.downloadHandler = void 0;
4
4
  const tslib_1 = require("tslib");
5
5
  /* eslint-disable no-await-in-loop */
6
+ /* eslint-disable no-param-reassign */
6
7
  const common_1 = require("@lightdash/common");
7
8
  const fs_1 = require("fs");
8
9
  const yaml = tslib_1.__importStar(require("js-yaml"));
@@ -11,56 +12,25 @@ const config_1 = require("../config");
11
12
  const globalState_1 = tslib_1.__importDefault(require("../globalState"));
12
13
  const apiClient_1 = require("./dbt/apiClient");
13
14
  const DOWNLOAD_FOLDER = 'lightdash';
14
- const downloadHandler = async (options) => {
15
- globalState_1.default.setVerbose(options.verbose);
16
- await (0, apiClient_1.checkLightdashVersion)();
17
- const config = await (0, config_1.getConfig)();
18
- if (!config.context?.apiKey || !config.context.serverUrl) {
19
- throw new common_1.AuthorizationError(`Not logged in. Run 'lightdash login --help'`);
20
- }
21
- const projectId = config.context.project;
22
- if (!projectId) {
23
- throw new Error('No project selected. Run lightdash config set-project');
24
- }
25
- const chartsAsCode = await (0, apiClient_1.lightdashApi)({
26
- method: 'GET',
27
- url: `/api/v1/projects/${projectId}/charts/code`,
28
- body: undefined,
29
- });
30
- console.info(`Downloading ${chartsAsCode.length} charts`);
31
- const outputDir = path.join(process.cwd(), DOWNLOAD_FOLDER);
32
- console.info(`Creating new path for files on ${outputDir} `);
33
- try {
34
- await fs_1.promises.mkdir(outputDir, { recursive: true });
35
- }
36
- catch (error) {
37
- // Directory already exists
38
- }
39
- for (const chart of chartsAsCode) {
40
- const chartPath = path.join(outputDir, `${chart.slug}.yml`);
41
- globalState_1.default.debug(`> Writing chart to ${chartPath}`);
42
- const chartYml = yaml.dump(chart, {
15
+ const dumpIntoFiles = async (folder, items) => {
16
+ const outputDir = path.join(process.cwd(), DOWNLOAD_FOLDER, folder);
17
+ console.info(`Writting ${items.length} ${folder} into ${outputDir}`);
18
+ // Make directory
19
+ const created = await fs_1.promises.mkdir(outputDir, { recursive: true });
20
+ if (created)
21
+ console.info(`Created new folder: ${outputDir} `);
22
+ for (const item of items) {
23
+ const itemPath = path.join(outputDir, `${item.slug}.yml`);
24
+ const chartYml = yaml.dump(item, {
43
25
  quotingType: '"',
44
26
  });
45
- await fs_1.promises.writeFile(chartPath, chartYml);
27
+ await fs_1.promises.writeFile(itemPath, chartYml);
46
28
  }
47
- // TODO delete files if chart don't exist ?*/
48
29
  };
49
- exports.downloadHandler = downloadHandler;
50
- const uploadHandler = async (options) => {
51
- globalState_1.default.setVerbose(options.verbose);
52
- await (0, apiClient_1.checkLightdashVersion)();
53
- const config = await (0, config_1.getConfig)();
54
- if (!config.context?.apiKey || !config.context.serverUrl) {
55
- throw new common_1.AuthorizationError(`Not logged in. Run 'lightdash login --help'`);
56
- }
57
- const projectId = config.context.project;
58
- if (!projectId) {
59
- throw new Error('No project selected. Run lightdash config set-project');
60
- }
61
- const inputDir = path.join(process.cwd(), DOWNLOAD_FOLDER);
62
- console.info(`Reading charts from ${inputDir}`);
63
- const charts = [];
30
+ const readCodeFiles = async (folder) => {
31
+ const inputDir = path.join(process.cwd(), DOWNLOAD_FOLDER, folder);
32
+ console.info(`Reading ${folder} from ${inputDir}`);
33
+ const items = [];
64
34
  try {
65
35
  // Read all files from the lightdash directory
66
36
  const files = await fs_1.promises.readdir(inputDir);
@@ -69,91 +39,138 @@ const uploadHandler = async (options) => {
69
39
  for (const file of jsonFiles) {
70
40
  const filePath = path.join(inputDir, file);
71
41
  const fileContent = await fs_1.promises.readFile(filePath, 'utf-8');
72
- const chart = yaml.load(fileContent);
42
+ const item = yaml.load(fileContent);
73
43
  const fileUpdatedAt = (await fs_1.promises.stat(filePath)).mtime;
74
44
  // We override the updatedAt to the file's updatedAt
75
45
  // in case there were some changes made locally
76
46
  // do not override if the file was just created
77
- const downloadedAt = chart.downloadedAt
78
- ? new Date(chart.downloadedAt)
47
+ const downloadedAt = item.downloadedAt
48
+ ? new Date(item.downloadedAt)
79
49
  : undefined;
80
50
  const needsUpdating = downloadedAt &&
81
51
  Math.abs(fileUpdatedAt.getTime() - downloadedAt.getTime()) >
82
52
  30000;
83
- const locallyUpdatedChart = {
84
- ...chart,
85
- updatedAt: needsUpdating ? fileUpdatedAt : chart.updatedAt,
53
+ const locallyUpdatedItem = {
54
+ ...item,
55
+ updatedAt: needsUpdating ? fileUpdatedAt : item.updatedAt,
86
56
  needsUpdating: needsUpdating ?? true, // if downloadAt is not set, we force the update
87
57
  };
88
- charts.push(locallyUpdatedChart);
58
+ items.push(locallyUpdatedItem);
89
59
  }
90
60
  }
91
61
  catch (error) {
92
62
  if (error.code === 'ENOENT') {
93
- throw new Error(`Directory ${inputDir} not found. Run download command first.`);
63
+ console.error(`Directory ${inputDir} not found. Run download command first.`);
64
+ }
65
+ else {
66
+ console.error(`Error reading ${inputDir}: ${error}`);
94
67
  }
95
68
  throw error;
96
69
  }
97
- console.info(`Found ${charts.length} chart files`);
98
- let created = 0;
99
- let updated = 0;
100
- let deleted = 0;
101
- let skipped = 0;
102
- let spacesCreated = 0;
103
- let spacesUpdated = 0;
104
- try {
105
- for (const chart of charts) {
106
- if (!chart.needsUpdating) {
107
- globalState_1.default.debug(`Skipping chart "${chart.slug}" with no local changes`);
108
- skipped += 1;
70
+ return items;
71
+ };
72
+ const downloadHandler = async (options) => {
73
+ globalState_1.default.setVerbose(options.verbose);
74
+ await (0, apiClient_1.checkLightdashVersion)();
75
+ const config = await (0, config_1.getConfig)();
76
+ if (!config.context?.apiKey || !config.context.serverUrl) {
77
+ throw new common_1.AuthorizationError(`Not logged in. Run 'lightdash login --help'`);
78
+ }
79
+ const projectId = config.context.project;
80
+ if (!projectId) {
81
+ throw new Error('No project selected. Run lightdash config set-project');
82
+ }
83
+ // Download charts
84
+ globalState_1.default.debug('Downloading charts');
85
+ const chartsAsCode = await (0, apiClient_1.lightdashApi)({
86
+ method: 'GET',
87
+ url: `/api/v1/projects/${projectId}/charts/code`,
88
+ body: undefined,
89
+ });
90
+ await dumpIntoFiles('charts', chartsAsCode);
91
+ // Download dashboards
92
+ globalState_1.default.debug('Downloading dashboards');
93
+ const dashboardsAsCode = await (0, apiClient_1.lightdashApi)({
94
+ method: 'GET',
95
+ url: `/api/v1/projects/${projectId}/dashboards/code`,
96
+ body: undefined,
97
+ });
98
+ await dumpIntoFiles('dashboards', dashboardsAsCode);
99
+ // TODO delete files if chart don't exist ?*/
100
+ };
101
+ exports.downloadHandler = downloadHandler;
102
+ const storeUploadChanges = (changes, promoteChanges) => {
103
+ const getPromoteChanges = (resource) => {
104
+ const promotions = promoteChanges[resource];
105
+ return promotions.reduce((acc, promoteChange) => {
106
+ const action = promoteChange.action === common_1.PromotionAction.NO_CHANGES
107
+ ? 'skipped'
108
+ : promoteChange.action;
109
+ const key = `${resource} ${action}`;
110
+ acc[key] = (acc[key] ?? 0) + 1;
111
+ return acc;
112
+ }, {});
113
+ };
114
+ const updatedChanges = {
115
+ ...changes,
116
+ };
117
+ ['spaces', 'charts', 'dashboards'].forEach((resource) => {
118
+ const resourceChanges = getPromoteChanges(resource);
119
+ Object.entries(resourceChanges).forEach(([key, value]) => {
120
+ updatedChanges[key] = (updatedChanges[key] ?? 0) + value;
121
+ });
122
+ });
123
+ return updatedChanges;
124
+ };
125
+ const logUploadChanges = (changes) => {
126
+ Object.entries(changes).forEach(([key, value]) => {
127
+ console.info(`Total ${key}: ${value} `);
128
+ });
129
+ };
130
+ const upsertResources = async (type, projectId, changes, force) => {
131
+ const items = await readCodeFiles(type);
132
+ console.info(`Found ${items.length} ${type} files`);
133
+ for (const item of items) {
134
+ // If a chart fails to update, we keep updating the rest
135
+ try {
136
+ if (!force && !item.needsUpdating) {
137
+ globalState_1.default.debug(`Skipping ${type} "${item.slug}" with no local changes`);
138
+ changes[`${type} skipped`] =
139
+ (changes[`${type} skipped`] ?? 0) + 1;
109
140
  // eslint-disable-next-line no-continue
110
141
  continue;
111
142
  }
112
- globalState_1.default.debug(`Upserting chart ${chart.slug}`);
113
- const chartData = await (0, apiClient_1.lightdashApi)({
143
+ globalState_1.default.debug(`Upserting ${type} ${item.slug}`);
144
+ const upsertData = await (0, apiClient_1.lightdashApi)({
114
145
  method: 'POST',
115
- url: `/api/v1/projects/${projectId}/charts/${chart.slug}/code`,
116
- body: JSON.stringify(chart),
146
+ url: `/api/v1/projects/${projectId}/${type}/${item.slug}/code`,
147
+ body: JSON.stringify(item),
117
148
  });
118
- globalState_1.default.debug(`Chart "${chart.name}": ${chartData.charts[0].action}`);
119
- switch (chartData.spaces[0].action) {
120
- case common_1.PromotionAction.CREATE:
121
- spacesCreated += 1;
122
- break;
123
- case common_1.PromotionAction.UPDATE:
124
- spacesUpdated += 1;
125
- break;
126
- default:
127
- // ignore the rest
128
- }
129
- switch (chartData.charts[0].action) {
130
- case common_1.PromotionAction.CREATE:
131
- created += 1;
132
- break;
133
- case common_1.PromotionAction.UPDATE:
134
- updated += 1;
135
- break;
136
- case common_1.PromotionAction.DELETE:
137
- deleted += 1;
138
- break;
139
- case common_1.PromotionAction.NO_CHANGES:
140
- skipped += 1;
141
- break;
142
- default:
143
- globalState_1.default.debug(`Unknown action: ${chartData.charts[0].action}`);
144
- break;
145
- }
149
+ globalState_1.default.debug(`${type} "${item.name}": ${upsertData[type]?.[0].action}`);
150
+ changes = storeUploadChanges(changes, upsertData);
151
+ }
152
+ catch (error) {
153
+ changes[`${type} with errors`] =
154
+ (changes[`${type} with errors`] ?? 0) + 1;
155
+ console.error(`Error upserting ${type}`, error);
146
156
  }
147
157
  }
148
- catch (error) {
149
- console.error('Error upserting chart', error);
158
+ return changes;
159
+ };
160
+ const uploadHandler = async (options) => {
161
+ globalState_1.default.setVerbose(options.verbose);
162
+ await (0, apiClient_1.checkLightdashVersion)();
163
+ const config = await (0, config_1.getConfig)();
164
+ if (!config.context?.apiKey || !config.context.serverUrl) {
165
+ throw new common_1.AuthorizationError(`Not logged in. Run 'lightdash login --help'`);
166
+ }
167
+ const projectId = config.context.project;
168
+ if (!projectId) {
169
+ throw new Error('No project selected. Run lightdash config set-project');
150
170
  }
151
- console.info(`Total charts created: ${created} `);
152
- console.info(`Total charts updated: ${updated} `);
153
- console.info(`Total charts skipped: ${skipped} `);
154
- if (deleted > 0)
155
- console.info(`Total charts deleted: ${deleted} `); // We should not delete charts from the CLI
156
- console.info(`Total spaces created: ${spacesCreated} `);
157
- console.info(`Total spaces updated: ${spacesUpdated} `);
171
+ let changes = {};
172
+ changes = await upsertResources('charts', projectId, changes, options.force);
173
+ changes = await upsertResources('dashboards', projectId, changes, options.force);
174
+ logUploadChanges(changes);
158
175
  };
159
176
  exports.uploadHandler = uploadHandler;
package/dist/index.js CHANGED
@@ -199,6 +199,7 @@ commander_1.program
199
199
  .command('upload')
200
200
  .description('Uploads charts and dashboards as code')
201
201
  .option('--verbose', undefined, false)
202
+ .option('--force', 'Force upload even if local files have not changed, use this when you want to upload files to a new project', false)
202
203
  .action(download_1.uploadHandler);
203
204
  commander_1.program
204
205
  .command('deploy')
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@lightdash/cli",
3
- "version": "0.1390.1",
3
+ "version": "0.1392.0",
4
4
  "license": "MIT",
5
5
  "bin": {
6
6
  "lightdash": "dist/index.js"
@@ -11,8 +11,8 @@
11
11
  ],
12
12
  "dependencies": {
13
13
  "@actions/core": "^1.11.1",
14
- "@lightdash/common": "^0.1390.1",
15
- "@lightdash/warehouses": "^0.1390.1",
14
+ "@lightdash/common": "^0.1392.0",
15
+ "@lightdash/warehouses": "^0.1392.0",
16
16
  "@types/columnify": "^1.5.1",
17
17
  "ajv": "^8.11.0",
18
18
  "ajv-formats": "^2.1.1",