@lightdash/cli 0.2203.0 → 0.2205.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/handlers/compile.d.ts +2 -1
- package/dist/handlers/compile.d.ts.map +1 -1
- package/dist/handlers/compile.js +94 -57
- package/dist/handlers/createProject.d.ts.map +1 -1
- package/dist/handlers/createProject.js +8 -4
- package/dist/handlers/deploy.d.ts.map +1 -1
- package/dist/handlers/deploy.js +15 -9
- package/dist/handlers/lint/ajvToSarif.d.ts +1 -1
- package/dist/handlers/lint/ajvToSarif.d.ts.map +1 -1
- package/dist/handlers/lint.d.ts.map +1 -1
- package/dist/handlers/lint.js +20 -2
- package/dist/index.js +2 -2
- package/dist/lightdash/loader.d.ts +18 -0
- package/dist/lightdash/loader.d.ts.map +1 -0
- package/dist/lightdash/loader.js +83 -0
- package/package.json +3 -3
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { Explore, ExploreError } from '@lightdash/common';
|
|
1
2
|
import { DbtCompileOptions } from './dbt/compile';
|
|
2
3
|
export type CompileHandlerOptions = DbtCompileOptions & {
|
|
3
4
|
projectDir: string;
|
|
@@ -9,6 +10,6 @@ export type CompileHandlerOptions = DbtCompileOptions & {
|
|
|
9
10
|
startOfWeek?: number;
|
|
10
11
|
warehouseCredentials?: boolean;
|
|
11
12
|
};
|
|
12
|
-
export declare const compile: (options: CompileHandlerOptions) => Promise<(
|
|
13
|
+
export declare const compile: (options: CompileHandlerOptions) => Promise<(Explore | ExploreError)[]>;
|
|
13
14
|
export declare const compileHandler: (originalOptions: CompileHandlerOptions) => Promise<void>;
|
|
14
15
|
//# sourceMappingURL=compile.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"compile.d.ts","sourceRoot":"","sources":["../../src/handlers/compile.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"compile.d.ts","sourceRoot":"","sources":["../../src/handlers/compile.ts"],"names":[],"mappings":"AAAA,OAAO,EAMH,OAAO,EACP,YAAY,EAUf,MAAM,mBAAmB,CAAC;AAY3B,OAAO,EAAE,iBAAiB,EAA8B,MAAM,eAAe,CAAC;AAI9E,MAAM,MAAM,qBAAqB,GAAG,iBAAiB,GAAG;IACpD,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,IAAI,EAAE,MAAM,GAAG,SAAS,CAAC;IACzB,OAAO,EAAE,OAAO,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,oBAAoB,CAAC,EAAE,OAAO,CAAC;CAClC,CAAC;AAyDF,eAAO,MAAM,OAAO,YAAmB,qBAAqB,wCAmL3D,CAAC;AACF,eAAO,MAAM,cAAc,oBACN,qBAAqB,kBAuBzC,CAAC"}
|
package/dist/handlers/compile.js
CHANGED
|
@@ -12,10 +12,36 @@ const manifest_1 = require("../dbt/manifest");
|
|
|
12
12
|
const validation_1 = require("../dbt/validation");
|
|
13
13
|
const globalState_1 = tslib_1.__importDefault(require("../globalState"));
|
|
14
14
|
const lightdash_config_1 = require("../lightdash-config");
|
|
15
|
+
const loader_1 = require("../lightdash/loader");
|
|
15
16
|
const styles = tslib_1.__importStar(require("../styles"));
|
|
16
17
|
const compile_1 = require("./dbt/compile");
|
|
17
18
|
const getDbtVersion_1 = require("./dbt/getDbtVersion");
|
|
18
19
|
const getWarehouseClient_1 = tslib_1.__importDefault(require("./dbt/getWarehouseClient"));
|
|
20
|
+
const getExploresFromLightdashYmlProject = async (projectDir, lightdashProjectConfig, startOfWeek) => {
|
|
21
|
+
// Try to load Lightdash YAML models
|
|
22
|
+
const lightdashModels = await (0, loader_1.loadLightdashModels)(projectDir);
|
|
23
|
+
if (lightdashModels.length === 0) {
|
|
24
|
+
return null; // No Lightdash models, use dbt path
|
|
25
|
+
}
|
|
26
|
+
globalState_1.default.debug(`> Found ${lightdashModels.length} Lightdash YAML models`);
|
|
27
|
+
if (!lightdashProjectConfig.warehouse?.type) {
|
|
28
|
+
throw new common_1.ParseError('Lightdash models found but no warehouse type specified in lightdash.config.yml.\n' +
|
|
29
|
+
'Add a warehouse section:\n' +
|
|
30
|
+
'warehouse:\n' +
|
|
31
|
+
' type: postgres # or bigquery, snowflake, redshift, databricks, trino, clickhouse');
|
|
32
|
+
}
|
|
33
|
+
const adapterType = lightdashProjectConfig.warehouse.type;
|
|
34
|
+
globalState_1.default.debug(`> Using adapter type from lightdash.config.yml: ${adapterType}`);
|
|
35
|
+
// Convert Lightdash models to DbtModelNode format
|
|
36
|
+
const validModels = (0, common_1.convertLightdashModelsToDbtModels)(lightdashModels);
|
|
37
|
+
if (validModels.length === 0) {
|
|
38
|
+
return null;
|
|
39
|
+
}
|
|
40
|
+
globalState_1.default.debug('> Skipping warehouse catalog (types in YAML)');
|
|
41
|
+
const warehouseSqlBuilder = (0, warehouses_1.warehouseSqlBuilderFromType)(adapterType, startOfWeek);
|
|
42
|
+
const validExplores = await (0, common_1.convertExplores)(validModels, false, warehouseSqlBuilder.getAdapterType(), [], warehouseSqlBuilder, lightdashProjectConfig);
|
|
43
|
+
return validExplores;
|
|
44
|
+
};
|
|
19
45
|
const compile = async (options) => {
|
|
20
46
|
const dbtVersion = await (0, getDbtVersion_1.getDbtVersion)();
|
|
21
47
|
globalState_1.default.debug(`> dbt version ${dbtVersion.verboseVersion}`);
|
|
@@ -32,64 +58,74 @@ const compile = async (options) => {
|
|
|
32
58
|
});
|
|
33
59
|
const absoluteProjectPath = path_1.default.resolve(options.projectDir);
|
|
34
60
|
globalState_1.default.debug(`> Compiling with project dir ${absoluteProjectPath}`);
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
targetPath: options.targetPath,
|
|
38
|
-
});
|
|
39
|
-
const compiledModelIds = await (0, compile_1.maybeCompileModelsAndJoins)({ targetDir: context.targetDir }, options);
|
|
40
|
-
const manifest = await (0, manifest_1.loadManifest)({ targetDir: context.targetDir });
|
|
41
|
-
const manifestVersion = (0, common_1.getDbtManifestVersion)(manifest);
|
|
42
|
-
const manifestModels = (0, common_1.getModelsFromManifest)(manifest);
|
|
43
|
-
const compiledModels = (0, common_1.getCompiledModels)(manifestModels, compiledModelIds);
|
|
44
|
-
const adapterType = manifest.metadata.adapter_type;
|
|
45
|
-
const { valid: validModels, invalid: failedExplores } = await (0, validation_1.validateDbtModel)(adapterType, manifestVersion, compiledModels);
|
|
46
|
-
if (failedExplores.length > 0) {
|
|
47
|
-
const errors = failedExplores.map((failedExplore) => failedExplore.errors.map((error) => `- ${failedExplore.name}: ${error.message}\n`));
|
|
48
|
-
console.error(styles.warning(`Found ${failedExplores.length} errors when validating dbt models:
|
|
49
|
-
${errors.join('')}`));
|
|
50
|
-
}
|
|
51
|
-
// Skipping assumes yml has the field types.
|
|
52
|
-
let catalog = {};
|
|
53
|
-
if (!options.skipWarehouseCatalog) {
|
|
54
|
-
const { warehouseClient } = await (0, getWarehouseClient_1.default)({
|
|
55
|
-
isDbtCloudCLI: dbtVersion.isDbtCloudCLI,
|
|
56
|
-
profilesDir: options.profilesDir,
|
|
57
|
-
profile: options.profile || context.profileName,
|
|
58
|
-
target: options.target,
|
|
59
|
-
startOfWeek: options.startOfWeek,
|
|
60
|
-
});
|
|
61
|
-
globalState_1.default.debug('> Fetching warehouse catalog');
|
|
62
|
-
catalog = await warehouseClient.getCatalog((0, common_1.getSchemaStructureFromDbtModels)(validModels));
|
|
63
|
-
}
|
|
64
|
-
else {
|
|
65
|
-
globalState_1.default.debug('> Skipping warehouse catalog');
|
|
66
|
-
}
|
|
67
|
-
const validModelsWithTypes = (0, common_1.attachTypesToModels)(validModels, catalog, false);
|
|
68
|
-
if (!(0, common_1.isSupportedDbtAdapter)(manifest.metadata)) {
|
|
69
|
-
await analytics_1.LightdashAnalytics.track({
|
|
70
|
-
event: 'compile.error',
|
|
71
|
-
properties: {
|
|
72
|
-
executionId,
|
|
73
|
-
dbtVersion: dbtVersion.verboseVersion,
|
|
74
|
-
error: `Dbt adapter ${manifest.metadata.adapter_type} is not supported`,
|
|
75
|
-
},
|
|
76
|
-
});
|
|
77
|
-
throw new common_1.ParseError(`Dbt adapter ${manifest.metadata.adapter_type} is not supported`);
|
|
78
|
-
}
|
|
79
|
-
globalState_1.default.debug(`> Converting explores with adapter: ${manifest.metadata.adapter_type}`);
|
|
80
|
-
globalState_1.default.debug(`> Loading lightdash project config from ${absoluteProjectPath}`);
|
|
61
|
+
// Load Lightdash Project
|
|
62
|
+
// Load Lightdash project config
|
|
81
63
|
const lightdashProjectConfig = await (0, lightdash_config_1.readAndLoadLightdashProjectConfig)(absoluteProjectPath);
|
|
82
64
|
globalState_1.default.debug(`> Loaded lightdash project config`);
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
65
|
+
// Try lightdash project compile
|
|
66
|
+
let explores = null;
|
|
67
|
+
let dbtMetrics = null;
|
|
68
|
+
explores = await getExploresFromLightdashYmlProject(absoluteProjectPath, lightdashProjectConfig, options.startOfWeek);
|
|
69
|
+
// Load dbt Project
|
|
70
|
+
if (explores === null) {
|
|
71
|
+
const context = await (0, context_1.getDbtContext)({
|
|
72
|
+
projectDir: absoluteProjectPath,
|
|
73
|
+
targetPath: options.targetPath,
|
|
74
|
+
});
|
|
75
|
+
const compiledModelIds = await (0, compile_1.maybeCompileModelsAndJoins)({ targetDir: context.targetDir }, options);
|
|
76
|
+
const manifest = await (0, manifest_1.loadManifest)({ targetDir: context.targetDir });
|
|
77
|
+
const manifestVersion = (0, common_1.getDbtManifestVersion)(manifest);
|
|
78
|
+
const manifestModels = (0, common_1.getModelsFromManifest)(manifest);
|
|
79
|
+
const compiledModels = (0, common_1.getCompiledModels)(manifestModels, compiledModelIds);
|
|
80
|
+
const adapterType = manifest.metadata.adapter_type;
|
|
81
|
+
const { valid: validModels, invalid: failedExplores } = await (0, validation_1.validateDbtModel)(adapterType, manifestVersion, compiledModels);
|
|
82
|
+
if (failedExplores.length > 0) {
|
|
83
|
+
const errors = failedExplores.map((failedExplore) => failedExplore.errors.map((error) => `- ${failedExplore.name}: ${error.message}\n`));
|
|
84
|
+
console.error(styles.warning(`Found ${failedExplores.length} errors when validating dbt models:
|
|
85
|
+
${errors.join('')}`));
|
|
86
|
+
}
|
|
87
|
+
// Skipping assumes yml has the field types.
|
|
88
|
+
let catalog = {};
|
|
89
|
+
if (!options.skipWarehouseCatalog) {
|
|
90
|
+
const { warehouseClient } = await (0, getWarehouseClient_1.default)({
|
|
91
|
+
isDbtCloudCLI: dbtVersion.isDbtCloudCLI,
|
|
92
|
+
profilesDir: options.profilesDir,
|
|
93
|
+
profile: options.profile || context.profileName,
|
|
94
|
+
target: options.target,
|
|
95
|
+
startOfWeek: options.startOfWeek,
|
|
96
|
+
});
|
|
97
|
+
globalState_1.default.debug('> Fetching warehouse catalog');
|
|
98
|
+
catalog = await warehouseClient.getCatalog((0, common_1.getSchemaStructureFromDbtModels)(validModels));
|
|
99
|
+
}
|
|
100
|
+
else {
|
|
101
|
+
globalState_1.default.debug('> Skipping warehouse catalog');
|
|
102
|
+
}
|
|
103
|
+
const validModelsWithTypes = (0, common_1.attachTypesToModels)(validModels, catalog, false);
|
|
104
|
+
if (!(0, common_1.isSupportedDbtAdapter)(manifest.metadata)) {
|
|
105
|
+
await analytics_1.LightdashAnalytics.track({
|
|
106
|
+
event: 'compile.error',
|
|
107
|
+
properties: {
|
|
108
|
+
executionId,
|
|
109
|
+
dbtVersion: dbtVersion.verboseVersion,
|
|
110
|
+
error: `Dbt adapter ${manifest.metadata.adapter_type} is not supported`,
|
|
111
|
+
},
|
|
112
|
+
});
|
|
113
|
+
throw new common_1.ParseError(`Dbt adapter ${manifest.metadata.adapter_type} is not supported`);
|
|
114
|
+
}
|
|
115
|
+
globalState_1.default.debug(`> Converting explores with adapter: ${manifest.metadata.adapter_type}`);
|
|
116
|
+
globalState_1.default.debug(`> Loading lightdash project config from ${absoluteProjectPath}`);
|
|
117
|
+
const warehouseSqlBuilder = (0, warehouses_1.warehouseSqlBuilderFromType)(adapterType, options.startOfWeek);
|
|
118
|
+
const validExplores = await (0, common_1.convertExplores)(validModelsWithTypes, false, manifest.metadata.adapter_type, [
|
|
119
|
+
common_1.DbtManifestVersion.V10,
|
|
120
|
+
common_1.DbtManifestVersion.V11,
|
|
121
|
+
common_1.DbtManifestVersion.V12,
|
|
122
|
+
].includes(manifestVersion)
|
|
123
|
+
? []
|
|
124
|
+
: Object.values(manifest.metrics), warehouseSqlBuilder, lightdashProjectConfig);
|
|
125
|
+
console.error('');
|
|
126
|
+
explores = [...validExplores, ...failedExplores];
|
|
127
|
+
dbtMetrics = manifest.metrics;
|
|
128
|
+
}
|
|
93
129
|
explores.forEach((e) => {
|
|
94
130
|
const status = (0, common_1.isExploreError)(e)
|
|
95
131
|
? styles.error('ERROR')
|
|
@@ -102,13 +138,14 @@ ${errors.join('')}`));
|
|
|
102
138
|
console.error('');
|
|
103
139
|
const errors = explores.filter((e) => (0, common_1.isExploreError)(e)).length;
|
|
104
140
|
console.error(`Compiled ${explores.length} explores, SUCCESS=${explores.length - errors} ERRORS=${errors}`);
|
|
141
|
+
const metricsCount = dbtMetrics === null ? 0 : Object.values(dbtMetrics.metrics).length;
|
|
105
142
|
await analytics_1.LightdashAnalytics.track({
|
|
106
143
|
event: 'compile.completed',
|
|
107
144
|
properties: {
|
|
108
145
|
executionId,
|
|
109
146
|
explores: explores.length,
|
|
110
147
|
errors,
|
|
111
|
-
dbtMetrics:
|
|
148
|
+
dbtMetrics: metricsCount,
|
|
112
149
|
dbtVersion: dbtVersion.verboseVersion,
|
|
113
150
|
},
|
|
114
151
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"createProject.d.ts","sourceRoot":"","sources":["../../src/handlers/createProject.ts"],"names":[],"mappings":"AAAA,OAAO,EAEH,+BAA+B,EAE/B,WAAW,EAEX,KAAK,uBAAuB,EAG/B,MAAM,mBAAmB,CAAC;AA2D3B,eAAO,MAAM,kCAAkC,SACrC,MAAM,KACb,OAAO,CAAC,MAAM,CAyChB,CAAC;AAEF,KAAK,oBAAoB,GAAG;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,IAAI,EAAE,WAAW,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,kBAAkB,CAAC,EAAE,+BAA+B,CAAC;IACrD,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,UAAU,CAAC,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,eAAO,MAAM,aAAa,YACb,oBAAoB,KAC9B,OAAO,CAAC,uBAAuB,GAAG,SAAS,
|
|
1
|
+
{"version":3,"file":"createProject.d.ts","sourceRoot":"","sources":["../../src/handlers/createProject.ts"],"names":[],"mappings":"AAAA,OAAO,EAEH,+BAA+B,EAE/B,WAAW,EAEX,KAAK,uBAAuB,EAG/B,MAAM,mBAAmB,CAAC;AA2D3B,eAAO,MAAM,kCAAkC,SACrC,MAAM,KACb,OAAO,CAAC,MAAM,CAyChB,CAAC;AAEF,KAAK,oBAAoB,GAAG;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,IAAI,EAAE,WAAW,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,kBAAkB,CAAC,EAAE,+BAA+B,CAAC;IACrD,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,UAAU,CAAC,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,eAAO,MAAM,aAAa,YACb,oBAAoB,KAC9B,OAAO,CAAC,uBAAuB,GAAG,SAAS,CAkI7C,CAAC"}
|
|
@@ -94,16 +94,16 @@ const createProject = async (options) => {
|
|
|
94
94
|
}
|
|
95
95
|
const dbtVersion = await (0, getDbtVersion_1.getDbtVersion)();
|
|
96
96
|
const absoluteProjectPath = path_1.default.resolve(options.projectDir);
|
|
97
|
-
const context = await (0, context_1.getDbtContext)({
|
|
98
|
-
projectDir: absoluteProjectPath,
|
|
99
|
-
targetPath: options.targetPath,
|
|
100
|
-
});
|
|
101
97
|
let targetName;
|
|
102
98
|
let credentials;
|
|
103
99
|
// If using organization credentials, don't load warehouse credentials from profiles
|
|
104
100
|
if (organizationWarehouseCredentialsUuid) {
|
|
105
101
|
globalState_1.default.debug(`> Using organization warehouse credentials: ${options.organizationCredentials}`);
|
|
106
102
|
// Still need to get target name for dbt connection
|
|
103
|
+
const context = await (0, context_1.getDbtContext)({
|
|
104
|
+
projectDir: absoluteProjectPath,
|
|
105
|
+
targetPath: options.targetPath,
|
|
106
|
+
});
|
|
107
107
|
globalState_1.default.debug(`> Using profiles dir ${options.profilesDir} and profile ${options.profile || context.profileName}`);
|
|
108
108
|
targetName = await (0, getDbtProfileTargetName_1.default)({
|
|
109
109
|
isDbtCloudCLI: dbtVersion.isDbtCloudCLI,
|
|
@@ -117,6 +117,10 @@ const createProject = async (options) => {
|
|
|
117
117
|
globalState_1.default.debug('> Creating project without warehouse credentials');
|
|
118
118
|
}
|
|
119
119
|
else {
|
|
120
|
+
const context = await (0, context_1.getDbtContext)({
|
|
121
|
+
projectDir: absoluteProjectPath,
|
|
122
|
+
targetPath: options.targetPath,
|
|
123
|
+
});
|
|
120
124
|
globalState_1.default.debug(`> Using profiles dir ${options.profilesDir} and profile ${options.profile || context.profileName}`);
|
|
121
125
|
targetName = await (0, getDbtProfileTargetName_1.default)({
|
|
122
126
|
isDbtCloudCLI: dbtVersion.isDbtCloudCLI,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"deploy.d.ts","sourceRoot":"","sources":["../../src/handlers/deploy.ts"],"names":[],"mappings":"AAAA,OAAO,EAEH,OAAO,EACP,YAAY,
|
|
1
|
+
{"version":3,"file":"deploy.d.ts","sourceRoot":"","sources":["../../src/handlers/deploy.ts"],"names":[],"mappings":"AAAA,OAAO,EAEH,OAAO,EACP,YAAY,EAQf,MAAM,mBAAmB,CAAC;AAiB3B,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAGlD,KAAK,oBAAoB,GAAG,iBAAiB,GAAG;IAC5C,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,OAAO,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,MAAM,CAAC,EAAE,OAAO,GAAG,MAAM,CAAC;IAC1B,OAAO,EAAE,OAAO,CAAC;IACjB,YAAY,EAAE,OAAO,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,uBAAuB,CAAC,EAAE,MAAM,CAAC;CACpC,CAAC;AAEF,KAAK,UAAU,GAAG,oBAAoB,GAAG;IACrC,WAAW,EAAE,MAAM,CAAC;CACvB,CAAC;AAkCF,eAAO,MAAM,MAAM,aACL,CAAC,OAAO,GAAG,YAAY,CAAC,EAAE,WAC3B,UAAU,KACpB,OAAO,CAAC,IAAI,CA2Cd,CAAC;AA8FF,eAAO,MAAM,aAAa,oBAA2B,oBAAoB,kBA+ExE,CAAC"}
|
package/dist/handlers/deploy.js
CHANGED
|
@@ -70,15 +70,21 @@ exports.deploy = deploy;
|
|
|
70
70
|
const createNewProject = async (executionId, options) => {
|
|
71
71
|
console.error('');
|
|
72
72
|
const absoluteProjectPath = path_1.default.resolve(options.projectDir);
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
73
|
+
let defaultProjectName = 'My new Lightdash Project'; // TODO: improve
|
|
74
|
+
try {
|
|
75
|
+
const context = await (0, context_1.getDbtContext)({
|
|
76
|
+
projectDir: absoluteProjectPath,
|
|
77
|
+
targetPath: options.targetPath,
|
|
78
|
+
});
|
|
79
|
+
defaultProjectName = (0, common_1.friendlyName)(context.projectName);
|
|
80
|
+
}
|
|
81
|
+
catch (e) {
|
|
82
|
+
if (e instanceof common_1.ParseError) {
|
|
83
|
+
// stick with default name
|
|
84
|
+
}
|
|
85
|
+
}
|
|
81
86
|
// If interactive and no name provided, prompt for project name
|
|
87
|
+
let projectName = defaultProjectName;
|
|
82
88
|
if (options.create === true && process.env.CI !== 'true') {
|
|
83
89
|
const answers = await inquirer_1.default.prompt([
|
|
84
90
|
{
|
|
@@ -102,7 +108,7 @@ const createNewProject = async (executionId, options) => {
|
|
|
102
108
|
properties: {
|
|
103
109
|
executionId,
|
|
104
110
|
projectName,
|
|
105
|
-
isDefaultName:
|
|
111
|
+
isDefaultName: defaultProjectName === projectName,
|
|
106
112
|
},
|
|
107
113
|
});
|
|
108
114
|
try {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ajvToSarif.d.ts","sourceRoot":"","sources":["../../../src/handlers/lint/ajvToSarif.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,KAAK,CAAC;AAEvC,KAAK,WAAW,GAAG,GAAG,CAAC,MAAM,EAAE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC;AAEjE,KAAK,oBAAoB,GAAG;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,WAAW,EAAE,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,UAAU,EAAE,OAAO,GAAG,WAAW,CAAC;
|
|
1
|
+
{"version":3,"file":"ajvToSarif.d.ts","sourceRoot":"","sources":["../../../src/handlers/lint/ajvToSarif.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,KAAK,CAAC;AAEvC,KAAK,WAAW,GAAG,GAAG,CAAC,MAAM,EAAE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAAC,CAAC;AAEjE,KAAK,oBAAoB,GAAG;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,WAAW,EAAE,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B,UAAU,EAAE,OAAO,GAAG,WAAW,GAAG,OAAO,CAAC;CAC/C,CAAC;AAGF,MAAM,MAAM,QAAQ,GAAG;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,QAAQ,EAAE,CAAC;CACpB,CAAC;AAEF,KAAK,QAAQ,GAAG;IACZ,IAAI,EAAE;QACF,MAAM,EAAE;YACJ,IAAI,EAAE,MAAM,CAAC;YACb,OAAO,EAAE,MAAM,CAAC;YAChB,cAAc,CAAC,EAAE,MAAM,CAAC;YACxB,KAAK,CAAC,EAAE,SAAS,EAAE,CAAC;SACvB,CAAC;KACL,CAAC;IACF,OAAO,EAAE,WAAW,EAAE,CAAC;CAC1B,CAAC;AAEF,KAAK,SAAS,GAAG;IACb,EAAE,EAAE,MAAM,CAAC;IACX,gBAAgB,EAAE;QACd,IAAI,EAAE,MAAM,CAAC;KAChB,CAAC;IACF,eAAe,CAAC,EAAE;QACd,IAAI,EAAE,MAAM,CAAC;KAChB,CAAC;IACF,IAAI,CAAC,EAAE;QACH,IAAI,EAAE,MAAM,CAAC;KAChB,CAAC;CACL,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,OAAO,GAAG,SAAS,GAAG,MAAM,CAAC;IACpC,OAAO,EAAE;QACL,IAAI,EAAE,MAAM,CAAC;KAChB,CAAC;IACF,SAAS,EAAE,aAAa,EAAE,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACxC,CAAC;AAEF,KAAK,aAAa,GAAG;IACjB,gBAAgB,EAAE;QACd,gBAAgB,EAAE;YACd,GAAG,EAAE,MAAM,CAAC;SACf,CAAC;QACF,MAAM,EAAE;YACJ,SAAS,EAAE,MAAM,CAAC;YAClB,WAAW,EAAE,MAAM,CAAC;SACvB,CAAC;KACL,CAAC;CACL,CAAC;AAmJF;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,oBAAoB,EAAE,GAAG,QAAQ,CAwG3E"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"lint.d.ts","sourceRoot":"","sources":["../../src/handlers/lint.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"lint.d.ts","sourceRoot":"","sources":["../../src/handlers/lint.ts"],"names":[],"mappings":"AAcA,KAAK,WAAW,GAAG;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,MAAM,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC;CAC3B,CAAC;AAsOF,wBAAsB,WAAW,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAmGrE"}
|
package/dist/handlers/lint.js
CHANGED
|
@@ -12,6 +12,7 @@ const ajvToSarif_1 = require("./lint/ajvToSarif");
|
|
|
12
12
|
const sarifFormatter_1 = require("./lint/sarifFormatter");
|
|
13
13
|
const validateChartSchema = ajv_1.ajv.compile(common_1.chartAsCodeSchema);
|
|
14
14
|
const validateDashboardSchema = ajv_1.ajv.compile(common_1.dashboardAsCodeSchema);
|
|
15
|
+
const validateModelSchema = ajv_1.ajv.compile(common_1.modelAsCodeSchema);
|
|
15
16
|
/**
|
|
16
17
|
* Find all YAML and JSON files in a path (file or directory).
|
|
17
18
|
* If a file path is provided, returns it if it's a .yml/.yaml/.json file.
|
|
@@ -130,6 +131,23 @@ function validateFile(filePath) {
|
|
|
130
131
|
};
|
|
131
132
|
}
|
|
132
133
|
const dataObj = data;
|
|
134
|
+
// Check if this is a model (has type: "model", "model/v1beta", or "model/v1")
|
|
135
|
+
if (dataObj.type === 'model' ||
|
|
136
|
+
dataObj.type === 'model/v1beta' ||
|
|
137
|
+
dataObj.type === 'model/v1') {
|
|
138
|
+
const valid = validateModelSchema(data);
|
|
139
|
+
if (!valid && validateModelSchema.errors) {
|
|
140
|
+
return {
|
|
141
|
+
filePath,
|
|
142
|
+
valid: false,
|
|
143
|
+
errors: validateModelSchema.errors,
|
|
144
|
+
fileContent,
|
|
145
|
+
locationMap,
|
|
146
|
+
type: 'model',
|
|
147
|
+
};
|
|
148
|
+
}
|
|
149
|
+
return { filePath, valid: true, type: 'model' };
|
|
150
|
+
}
|
|
133
151
|
// Check if this is a chart (has version and metricQuery)
|
|
134
152
|
if (dataObj.version === 1 && dataObj.metricQuery && !dataObj.tiles) {
|
|
135
153
|
const valid = validateChartSchema(data);
|
|
@@ -199,7 +217,7 @@ async function lintHandler(options) {
|
|
|
199
217
|
const results = [];
|
|
200
218
|
for (const file of codeFiles) {
|
|
201
219
|
const result = validateFile(file);
|
|
202
|
-
// Only track Lightdash Code files (
|
|
220
|
+
// Only track Lightdash Code files (models, charts, dashboards)
|
|
203
221
|
if (result.type) {
|
|
204
222
|
results.push(result);
|
|
205
223
|
}
|
|
@@ -207,7 +225,7 @@ async function lintHandler(options) {
|
|
|
207
225
|
if (results.length === 0) {
|
|
208
226
|
if (outputFormat === 'cli') {
|
|
209
227
|
console.log(chalk_1.default.yellow('No Lightdash Code files found.'));
|
|
210
|
-
console.log(chalk_1.default.dim('
|
|
228
|
+
console.log(chalk_1.default.dim('Models must have type: model (or model/v1, model/v1beta), charts must have version: 1 + metricQuery, dashboards must have version: 1 + tiles'));
|
|
211
229
|
}
|
|
212
230
|
return;
|
|
213
231
|
}
|
package/dist/index.js
CHANGED
|
@@ -114,7 +114,7 @@ ${styles.bold('Examples:')}
|
|
|
114
114
|
${styles.title('⚡')}️lightdash ${styles.bold('dbt run')} -s mymodel ${styles.secondary('-- runs a single model and generates .yml')}
|
|
115
115
|
${styles.title('⚡')}️lightdash ${styles.bold('dbt run')} -s model1 model2 ${styles.secondary('-- runs multiple models and generates .yml')}
|
|
116
116
|
${styles.title('⚡')}️lightdash ${styles.bold('dbt run')} -s tag:sales ${styles.secondary('-- runs all models tagged as "sales" and generates .yml')}
|
|
117
|
-
${styles.title('⚡')}️lightdash ${styles.bold('dbt run')} -s +mymodel ${styles.secondary(
|
|
117
|
+
${styles.title('⚡')}️lightdash ${styles.bold('dbt run')} -s +mymodel ${styles.secondary('-- runs mymodel and its parents and generates .yml')}
|
|
118
118
|
`)
|
|
119
119
|
.option('--project-dir <path>', 'The directory of the dbt project', env_1.DEFAULT_DBT_PROJECT_DIR)
|
|
120
120
|
.option('--profiles-dir <path>', 'The directory of the dbt profiles', env_1.DEFAULT_DBT_PROFILES_DIR)
|
|
@@ -376,7 +376,7 @@ ${styles.bold('Examples:')}
|
|
|
376
376
|
.action(diagnostics_1.diagnosticsHandler);
|
|
377
377
|
commander_1.program
|
|
378
378
|
.command('lint')
|
|
379
|
-
.description('Validates Lightdash Code files (charts, dashboards) against JSON schemas')
|
|
379
|
+
.description('Validates Lightdash Code files (models, charts, dashboards) against JSON schemas')
|
|
380
380
|
.addHelpText('after', `
|
|
381
381
|
${styles.bold('Examples:')}
|
|
382
382
|
${styles.title('⚡')}️lightdash ${styles.bold('lint')} ${styles.secondary('-- validates all Lightdash Code files in current directory')}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Loader for Lightdash YAML model files
|
|
3
|
+
*/
|
|
4
|
+
import { type LightdashModel } from '@lightdash/common';
|
|
5
|
+
/**
|
|
6
|
+
* Load a single Lightdash YAML model file
|
|
7
|
+
*/
|
|
8
|
+
export declare function loadLightdashModel(filePath: string): Promise<LightdashModel>;
|
|
9
|
+
/**
|
|
10
|
+
* Find all Lightdash YAML model files in a directory
|
|
11
|
+
* Looks for files in the lightdash/models directory
|
|
12
|
+
*/
|
|
13
|
+
export declare function findLightdashModelFiles(projectDir: string): Promise<string[]>;
|
|
14
|
+
/**
|
|
15
|
+
* Load all Lightdash YAML models from a project directory
|
|
16
|
+
*/
|
|
17
|
+
export declare function loadLightdashModels(projectDir: string): Promise<LightdashModel[]>;
|
|
18
|
+
//# sourceMappingURL=loader.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"loader.d.ts","sourceRoot":"","sources":["../../src/lightdash/loader.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAc,KAAK,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAMpE;;GAEG;AACH,wBAAsB,kBAAkB,CACpC,QAAQ,EAAE,MAAM,GACjB,OAAO,CAAC,cAAc,CAAC,CA6BzB;AAED;;;GAGG;AACH,wBAAsB,uBAAuB,CACzC,UAAU,EAAE,MAAM,GACnB,OAAO,CAAC,MAAM,EAAE,CAAC,CA+BnB;AAED;;GAEG;AACH,wBAAsB,mBAAmB,CACrC,UAAU,EAAE,MAAM,GACnB,OAAO,CAAC,cAAc,EAAE,CAAC,CAwB3B"}
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* Loader for Lightdash YAML model files
|
|
4
|
+
*/
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.loadLightdashModel = loadLightdashModel;
|
|
7
|
+
exports.findLightdashModelFiles = findLightdashModelFiles;
|
|
8
|
+
exports.loadLightdashModels = loadLightdashModels;
|
|
9
|
+
const tslib_1 = require("tslib");
|
|
10
|
+
const common_1 = require("@lightdash/common");
|
|
11
|
+
const fs = tslib_1.__importStar(require("fs"));
|
|
12
|
+
const yaml = tslib_1.__importStar(require("js-yaml"));
|
|
13
|
+
const path = tslib_1.__importStar(require("path"));
|
|
14
|
+
const globalState_1 = tslib_1.__importDefault(require("../globalState"));
|
|
15
|
+
/**
|
|
16
|
+
* Load a single Lightdash YAML model file
|
|
17
|
+
*/
|
|
18
|
+
async function loadLightdashModel(filePath) {
|
|
19
|
+
try {
|
|
20
|
+
const fileContents = await fs.promises.readFile(filePath, 'utf8');
|
|
21
|
+
const parsed = yaml.load(fileContents);
|
|
22
|
+
// Basic validation
|
|
23
|
+
if (!parsed.type || !parsed.name || !parsed.dimensions) {
|
|
24
|
+
throw new common_1.ParseError(`Invalid Lightdash model in ${filePath}: must have type, name, and dimensions`);
|
|
25
|
+
}
|
|
26
|
+
if (!parsed.sql_from) {
|
|
27
|
+
throw new common_1.ParseError(`Invalid Lightdash model in ${filePath}: must have sql_from`);
|
|
28
|
+
}
|
|
29
|
+
return parsed;
|
|
30
|
+
}
|
|
31
|
+
catch (error) {
|
|
32
|
+
if (error instanceof common_1.ParseError) {
|
|
33
|
+
throw error;
|
|
34
|
+
}
|
|
35
|
+
throw new common_1.ParseError(`Failed to load Lightdash model from ${filePath}: ${error instanceof Error ? error.message : String(error)}`);
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Find all Lightdash YAML model files in a directory
|
|
40
|
+
* Looks for files in the lightdash/models directory
|
|
41
|
+
*/
|
|
42
|
+
async function findLightdashModelFiles(projectDir) {
|
|
43
|
+
const lightdashModelsDir = path.join(projectDir, 'lightdash', 'models');
|
|
44
|
+
if (!fs.existsSync(lightdashModelsDir)) {
|
|
45
|
+
globalState_1.default.debug(`No lightdash/models directory found at ${lightdashModelsDir}`);
|
|
46
|
+
return [];
|
|
47
|
+
}
|
|
48
|
+
const files = [];
|
|
49
|
+
async function walkDir(dir) {
|
|
50
|
+
const entries = await fs.promises.readdir(dir, { withFileTypes: true });
|
|
51
|
+
for await (const entry of entries) {
|
|
52
|
+
const fullPath = path.join(dir, entry.name);
|
|
53
|
+
if (entry.isDirectory()) {
|
|
54
|
+
await walkDir(fullPath);
|
|
55
|
+
}
|
|
56
|
+
else if (entry.isFile() &&
|
|
57
|
+
(entry.name.endsWith('.yml') || entry.name.endsWith('.yaml'))) {
|
|
58
|
+
files.push(fullPath);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
await walkDir(lightdashModelsDir);
|
|
63
|
+
return files;
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Load all Lightdash YAML models from a project directory
|
|
67
|
+
*/
|
|
68
|
+
async function loadLightdashModels(projectDir) {
|
|
69
|
+
const modelFiles = await findLightdashModelFiles(projectDir);
|
|
70
|
+
globalState_1.default.debug(`Found ${modelFiles.length} Lightdash model files in ${projectDir}`);
|
|
71
|
+
const models = [];
|
|
72
|
+
for await (const filePath of modelFiles) {
|
|
73
|
+
try {
|
|
74
|
+
const model = await loadLightdashModel(filePath);
|
|
75
|
+
models.push(model);
|
|
76
|
+
globalState_1.default.debug(`Loaded Lightdash model: ${model.name}`);
|
|
77
|
+
}
|
|
78
|
+
catch (error) {
|
|
79
|
+
console.error(`Warning: Failed to load ${filePath}: ${error instanceof Error ? error.message : String(error)}`);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
return models;
|
|
83
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@lightdash/cli",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2205.0",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"bin": {
|
|
6
6
|
"lightdash": "dist/index.js"
|
|
@@ -34,8 +34,8 @@
|
|
|
34
34
|
"unique-names-generator": "^4.7.1",
|
|
35
35
|
"uuid": "^11.0.3",
|
|
36
36
|
"yaml": "^2.7.0",
|
|
37
|
-
"@lightdash/
|
|
38
|
-
"@lightdash/
|
|
37
|
+
"@lightdash/common": "0.2205.0",
|
|
38
|
+
"@lightdash/warehouses": "0.2205.0"
|
|
39
39
|
},
|
|
40
40
|
"description": "Lightdash CLI tool",
|
|
41
41
|
"devDependencies": {
|