@duckcodeailabs/dql-cli 0.8.5 → 0.8.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -4
- package/dist/args.d.ts +1 -0
- package/dist/args.d.ts.map +1 -1
- package/dist/args.js +4 -0
- package/dist/args.js.map +1 -1
- package/dist/assets/dql-notebook/assets/index-CTmiMNUc.js +558 -0
- package/dist/assets/dql-notebook/index.html +1 -1
- package/dist/block-templates.d.ts +8 -0
- package/dist/block-templates.d.ts.map +1 -0
- package/dist/block-templates.js +60 -0
- package/dist/block-templates.js.map +1 -0
- package/dist/commands/build.test.js +1 -1
- package/dist/commands/build.test.js.map +1 -1
- package/dist/commands/doctor.d.ts.map +1 -1
- package/dist/commands/doctor.js +17 -1
- package/dist/commands/doctor.js.map +1 -1
- package/dist/commands/doctor.test.js +1 -1
- package/dist/commands/doctor.test.js.map +1 -1
- package/dist/commands/init.d.ts.map +1 -1
- package/dist/commands/init.js +73 -5
- package/dist/commands/init.js.map +1 -1
- package/dist/commands/init.test.js +82 -2
- package/dist/commands/init.test.js.map +1 -1
- package/dist/commands/new.test.js +7 -7
- package/dist/commands/new.test.js.map +1 -1
- package/dist/commands/notebook.d.ts.map +1 -1
- package/dist/commands/notebook.js +2 -2
- package/dist/commands/notebook.js.map +1 -1
- package/dist/commands/semantic.d.ts +2 -0
- package/dist/commands/semantic.d.ts.map +1 -1
- package/dist/commands/semantic.js +115 -1
- package/dist/commands/semantic.js.map +1 -1
- package/dist/index.js +18 -0
- package/dist/index.js.map +1 -1
- package/dist/local-runtime.d.ts +35 -0
- package/dist/local-runtime.d.ts.map +1 -1
- package/dist/local-runtime.js +1668 -45
- package/dist/local-runtime.js.map +1 -1
- package/dist/local-runtime.test.js +69 -1
- package/dist/local-runtime.test.js.map +1 -1
- package/dist/semantic-import.d.ts +127 -0
- package/dist/semantic-import.d.ts.map +1 -0
- package/dist/semantic-import.js +713 -0
- package/dist/semantic-import.js.map +1 -0
- package/dist/semantic-import.test.d.ts +2 -0
- package/dist/semantic-import.test.d.ts.map +1 -0
- package/dist/semantic-import.test.js +278 -0
- package/dist/semantic-import.test.js.map +1 -0
- package/package.json +9 -8
- package/dist/assets/dql-notebook/assets/index-Rushqlh8.js +0 -524
|
@@ -0,0 +1,713 @@
|
|
|
1
|
+
import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
|
2
|
+
import { dirname, join, resolve } from 'node:path';
|
|
3
|
+
import { CubejsProvider, DbtProvider, SnowflakeSemanticProvider, resolveRepoSource, } from '@duckcodeailabs/dql-core';
|
|
4
|
+
const MANIFEST_RELATIVE_PATH = 'semantic-layer/imports/manifest.json';
|
|
5
|
+
export async function performSemanticImport(opts) {
|
|
6
|
+
const targetProjectRoot = resolve(opts.targetProjectRoot);
|
|
7
|
+
const previousManifest = loadSemanticImportManifest(targetProjectRoot);
|
|
8
|
+
const previousManaged = new Set(previousManifest?.generatedFiles ?? []);
|
|
9
|
+
const source = resolveImportSource(opts.targetProjectRoot, opts.sourceConfig);
|
|
10
|
+
const layer = await loadLayerForImport(opts.provider, source.localPath, source.config, opts.executeQuery);
|
|
11
|
+
const warnings = [...source.warnings, ...collectImportWarnings(opts.provider, layer)];
|
|
12
|
+
const objects = collectObjects(layer);
|
|
13
|
+
const generatedFiles = [];
|
|
14
|
+
const manifestObjects = [];
|
|
15
|
+
for (const relPath of previousManaged) {
|
|
16
|
+
const absPath = join(targetProjectRoot, relPath);
|
|
17
|
+
if (existsSync(absPath)) {
|
|
18
|
+
rmSync(absPath, { force: true });
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
for (const object of objects) {
|
|
22
|
+
const normalizedDomain = normalizeDomain(object.domain);
|
|
23
|
+
const filePath = buildSemanticFilePath(object.kind, normalizedDomain, object.name);
|
|
24
|
+
const absPath = join(targetProjectRoot, filePath);
|
|
25
|
+
if (existsSync(absPath) && !previousManaged.has(filePath)) {
|
|
26
|
+
throw new Error(`Import conflict: ${filePath} already exists and is not managed by semantic import.`);
|
|
27
|
+
}
|
|
28
|
+
mkdirSync(dirname(absPath), { recursive: true });
|
|
29
|
+
writeFileSync(absPath, serializeSemanticObject(object), 'utf-8');
|
|
30
|
+
generatedFiles.push(filePath);
|
|
31
|
+
manifestObjects.push({
|
|
32
|
+
id: objectId(object.kind, object.name),
|
|
33
|
+
kind: object.kind,
|
|
34
|
+
name: object.name,
|
|
35
|
+
label: object.label,
|
|
36
|
+
domain: normalizedDomain,
|
|
37
|
+
cube: 'cube' in object ? object.cube : undefined,
|
|
38
|
+
filePath,
|
|
39
|
+
source: object.source,
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
const manifest = {
|
|
43
|
+
version: 1,
|
|
44
|
+
mode: 'imported',
|
|
45
|
+
provider: opts.provider,
|
|
46
|
+
importedAt: new Date().toISOString(),
|
|
47
|
+
source: {
|
|
48
|
+
projectPath: opts.sourceConfig.projectPath,
|
|
49
|
+
repoUrl: opts.sourceConfig.repoUrl,
|
|
50
|
+
branch: opts.sourceConfig.branch,
|
|
51
|
+
subPath: opts.sourceConfig.subPath,
|
|
52
|
+
connection: opts.sourceConfig.connection,
|
|
53
|
+
},
|
|
54
|
+
warnings,
|
|
55
|
+
generatedFiles: [...generatedFiles, MANIFEST_RELATIVE_PATH],
|
|
56
|
+
objects: manifestObjects,
|
|
57
|
+
};
|
|
58
|
+
const manifestPath = join(targetProjectRoot, MANIFEST_RELATIVE_PATH);
|
|
59
|
+
mkdirSync(dirname(manifestPath), { recursive: true });
|
|
60
|
+
writeFileSync(manifestPath, JSON.stringify(manifest, null, 2) + '\n', 'utf-8');
|
|
61
|
+
applyCanonicalSemanticConfig(targetProjectRoot);
|
|
62
|
+
return {
|
|
63
|
+
manifest,
|
|
64
|
+
counts: countObjects(manifestObjects),
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
export function syncSemanticImport(opts) {
|
|
68
|
+
const manifest = loadSemanticImportManifest(opts.targetProjectRoot);
|
|
69
|
+
if (!manifest) {
|
|
70
|
+
throw new Error('No semantic import manifest found. Run `dql semantic import <provider>` first.');
|
|
71
|
+
}
|
|
72
|
+
const sourceConfig = {
|
|
73
|
+
provider: manifest.provider,
|
|
74
|
+
projectPath: manifest.source.projectPath,
|
|
75
|
+
repoUrl: manifest.source.repoUrl,
|
|
76
|
+
branch: manifest.source.branch,
|
|
77
|
+
subPath: manifest.source.subPath,
|
|
78
|
+
connection: manifest.source.connection,
|
|
79
|
+
};
|
|
80
|
+
return performSemanticImport({
|
|
81
|
+
targetProjectRoot: opts.targetProjectRoot,
|
|
82
|
+
provider: manifest.provider,
|
|
83
|
+
sourceConfig,
|
|
84
|
+
executeQuery: opts.executeQuery,
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
export function loadSemanticImportManifest(projectRoot) {
|
|
88
|
+
const manifestPath = join(resolve(projectRoot), MANIFEST_RELATIVE_PATH);
|
|
89
|
+
if (!existsSync(manifestPath))
|
|
90
|
+
return null;
|
|
91
|
+
try {
|
|
92
|
+
return JSON.parse(readFileSync(manifestPath, 'utf-8'));
|
|
93
|
+
}
|
|
94
|
+
catch {
|
|
95
|
+
return null;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
export function buildSemanticTree(layer, manifest) {
|
|
99
|
+
const providerName = manifest?.provider ?? 'dql';
|
|
100
|
+
const domains = layer.listDomains();
|
|
101
|
+
const cubes = layer.listCubes();
|
|
102
|
+
const metrics = layer.listMetrics();
|
|
103
|
+
const dimensions = layer.listDimensions();
|
|
104
|
+
const hierarchies = layer.listHierarchies();
|
|
105
|
+
const segments = layer.listSegments();
|
|
106
|
+
const preAggregations = layer.listPreAggregations();
|
|
107
|
+
const domainNodes = domains.map((domain) => {
|
|
108
|
+
const domainCubes = cubes.filter((cube) => cube.domain === domain);
|
|
109
|
+
const looseMetrics = metrics.filter((metric) => metric.domain === domain && !metric.cube);
|
|
110
|
+
const looseDimensions = dimensions.filter((dimension) => dimension.domain === domain && !dimension.cube);
|
|
111
|
+
const domainHierarchies = hierarchies.filter((hierarchy) => hierarchy.domain === domain);
|
|
112
|
+
const cubeNodes = domainCubes.map((cube) => ({
|
|
113
|
+
id: objectId('cube', cube.name),
|
|
114
|
+
label: cube.label,
|
|
115
|
+
kind: 'cube',
|
|
116
|
+
count: cube.measures.length +
|
|
117
|
+
cube.dimensions.length +
|
|
118
|
+
cube.timeDimensions.length +
|
|
119
|
+
cube.segments.length +
|
|
120
|
+
cube.preAggregations.length,
|
|
121
|
+
meta: {
|
|
122
|
+
provider: cube.source?.provider ?? providerName,
|
|
123
|
+
domain: normalizeDomain(cube.domain),
|
|
124
|
+
cube: cube.name,
|
|
125
|
+
owner: cube.owner ?? null,
|
|
126
|
+
tags: (cube.tags ?? []).join(','),
|
|
127
|
+
table: cube.table,
|
|
128
|
+
},
|
|
129
|
+
children: [
|
|
130
|
+
buildGroupNode('metric', 'Measures', cube.measures.map((metric) => toLeaf('metric', metric.name, metric.label, {
|
|
131
|
+
provider: metric.source?.provider ?? providerName,
|
|
132
|
+
domain: normalizeDomain(metric.domain),
|
|
133
|
+
cube: metric.cube ?? cube.name,
|
|
134
|
+
owner: metric.owner ?? cube.owner ?? null,
|
|
135
|
+
tags: (metric.tags ?? []).join(','),
|
|
136
|
+
table: metric.table,
|
|
137
|
+
}))),
|
|
138
|
+
buildGroupNode('dimension', 'Dimensions', [...cube.dimensions, ...cube.timeDimensions].map((dimension) => toLeaf('dimension', dimension.name, dimension.label, {
|
|
139
|
+
provider: dimension.source?.provider ?? providerName,
|
|
140
|
+
domain: normalizeDomain(dimension.domain),
|
|
141
|
+
cube: dimension.cube ?? cube.name,
|
|
142
|
+
owner: dimension.owner ?? cube.owner ?? null,
|
|
143
|
+
tags: (dimension.tags ?? []).join(','),
|
|
144
|
+
table: dimension.table,
|
|
145
|
+
}))),
|
|
146
|
+
buildGroupNode('segment', 'Segments', cube.segments.map((segment) => toLeaf('segment', segment.name, segment.label, {
|
|
147
|
+
provider: segment.source?.provider ?? providerName,
|
|
148
|
+
domain: normalizeDomain(segment.domain),
|
|
149
|
+
cube: segment.cube || cube.name,
|
|
150
|
+
owner: segment.owner ?? cube.owner ?? null,
|
|
151
|
+
tags: (segment.tags ?? []).join(','),
|
|
152
|
+
}))),
|
|
153
|
+
buildGroupNode('pre_aggregation', 'Pre-aggregations', cube.preAggregations.map((preAggregation) => toLeaf('pre_aggregation', preAggregation.name, preAggregation.label, {
|
|
154
|
+
provider: preAggregation.source?.provider ?? providerName,
|
|
155
|
+
domain: normalizeDomain(preAggregation.domain),
|
|
156
|
+
cube: preAggregation.cube || cube.name,
|
|
157
|
+
owner: preAggregation.owner ?? cube.owner ?? null,
|
|
158
|
+
tags: (preAggregation.tags ?? []).join(','),
|
|
159
|
+
}))),
|
|
160
|
+
].filter((node) => Boolean(node)),
|
|
161
|
+
}));
|
|
162
|
+
const children = [...cubeNodes];
|
|
163
|
+
const looseNodes = [
|
|
164
|
+
buildGroupNode('metric', 'Metrics', looseMetrics.map((metric) => toLeaf('metric', metric.name, metric.label, {
|
|
165
|
+
provider: metric.source?.provider ?? providerName,
|
|
166
|
+
domain: normalizeDomain(metric.domain),
|
|
167
|
+
cube: metric.cube ?? null,
|
|
168
|
+
owner: metric.owner ?? null,
|
|
169
|
+
tags: (metric.tags ?? []).join(','),
|
|
170
|
+
table: metric.table,
|
|
171
|
+
}))),
|
|
172
|
+
buildGroupNode('dimension', 'Dimensions', looseDimensions.map((dimension) => toLeaf('dimension', dimension.name, dimension.label, {
|
|
173
|
+
provider: dimension.source?.provider ?? providerName,
|
|
174
|
+
domain: normalizeDomain(dimension.domain),
|
|
175
|
+
cube: dimension.cube ?? null,
|
|
176
|
+
owner: dimension.owner ?? null,
|
|
177
|
+
tags: (dimension.tags ?? []).join(','),
|
|
178
|
+
table: dimension.table,
|
|
179
|
+
}))),
|
|
180
|
+
buildGroupNode('hierarchy', 'Hierarchies', domainHierarchies.map((hierarchy) => toLeaf('hierarchy', hierarchy.name, hierarchy.label, {
|
|
181
|
+
provider: hierarchy.source?.provider ?? providerName,
|
|
182
|
+
domain: normalizeDomain(hierarchy.domain),
|
|
183
|
+
owner: hierarchy.owner ?? null,
|
|
184
|
+
tags: (hierarchy.tags ?? []).join(','),
|
|
185
|
+
}))),
|
|
186
|
+
buildGroupNode('segment', 'Segments', segments.filter((segment) => segment.domain === domain && !segment.cube).map((segment) => toLeaf('segment', segment.name, segment.label, {
|
|
187
|
+
provider: segment.source?.provider ?? providerName,
|
|
188
|
+
domain: normalizeDomain(segment.domain),
|
|
189
|
+
cube: segment.cube || null,
|
|
190
|
+
owner: segment.owner ?? null,
|
|
191
|
+
tags: (segment.tags ?? []).join(','),
|
|
192
|
+
}))),
|
|
193
|
+
buildGroupNode('pre_aggregation', 'Pre-aggregations', preAggregations.filter((preAggregation) => preAggregation.domain === domain && !preAggregation.cube).map((preAggregation) => toLeaf('pre_aggregation', preAggregation.name, preAggregation.label, {
|
|
194
|
+
provider: preAggregation.source?.provider ?? providerName,
|
|
195
|
+
domain: normalizeDomain(preAggregation.domain),
|
|
196
|
+
cube: preAggregation.cube || null,
|
|
197
|
+
owner: preAggregation.owner ?? null,
|
|
198
|
+
tags: (preAggregation.tags ?? []).join(','),
|
|
199
|
+
}))),
|
|
200
|
+
].filter((node) => Boolean(node));
|
|
201
|
+
children.push(...looseNodes);
|
|
202
|
+
return {
|
|
203
|
+
id: `domain:${domain}`,
|
|
204
|
+
label: domain,
|
|
205
|
+
kind: 'domain',
|
|
206
|
+
count: children.reduce((sum, node) => sum + (node.count ?? 0), 0),
|
|
207
|
+
meta: {
|
|
208
|
+
provider: providerName,
|
|
209
|
+
domain,
|
|
210
|
+
},
|
|
211
|
+
children,
|
|
212
|
+
};
|
|
213
|
+
});
|
|
214
|
+
return {
|
|
215
|
+
id: `provider:${manifest?.provider ?? 'dql'}`,
|
|
216
|
+
label: manifest?.provider ? `${manifest.provider} import` : 'semantic layer',
|
|
217
|
+
kind: 'provider',
|
|
218
|
+
count: domains.length,
|
|
219
|
+
meta: {
|
|
220
|
+
provider: providerName,
|
|
221
|
+
importedAt: manifest?.importedAt ?? null,
|
|
222
|
+
warnings: manifest?.warnings.length ?? 0,
|
|
223
|
+
},
|
|
224
|
+
children: domainNodes,
|
|
225
|
+
};
|
|
226
|
+
}
|
|
227
|
+
export function buildSemanticObjectDetail(layer, manifest, id) {
|
|
228
|
+
const [kind, ...rest] = id.split(':');
|
|
229
|
+
const name = rest.join(':');
|
|
230
|
+
const manifestObject = manifest?.objects.find((object) => object.id === id) ?? null;
|
|
231
|
+
const importedAt = manifest?.importedAt ?? null;
|
|
232
|
+
if (kind === 'cube') {
|
|
233
|
+
const cube = layer.getCube(name);
|
|
234
|
+
if (!cube)
|
|
235
|
+
return null;
|
|
236
|
+
return {
|
|
237
|
+
id,
|
|
238
|
+
kind: 'cube',
|
|
239
|
+
name: cube.name,
|
|
240
|
+
label: cube.label,
|
|
241
|
+
description: cube.description,
|
|
242
|
+
domain: cube.domain || normalizeDomain(undefined),
|
|
243
|
+
table: cube.table,
|
|
244
|
+
sql: cube.sql,
|
|
245
|
+
tags: cube.tags ?? [],
|
|
246
|
+
owner: cube.owner ?? null,
|
|
247
|
+
source: cube.source ?? manifestObject?.source ?? null,
|
|
248
|
+
filePath: manifestObject?.filePath ?? null,
|
|
249
|
+
importedAt,
|
|
250
|
+
joins: cube.joins,
|
|
251
|
+
};
|
|
252
|
+
}
|
|
253
|
+
if (kind === 'metric') {
|
|
254
|
+
const metric = layer.getMetric(name);
|
|
255
|
+
if (!metric)
|
|
256
|
+
return null;
|
|
257
|
+
return {
|
|
258
|
+
id,
|
|
259
|
+
kind: 'metric',
|
|
260
|
+
name: metric.name,
|
|
261
|
+
label: metric.label,
|
|
262
|
+
description: metric.description,
|
|
263
|
+
domain: normalizeDomain(metric.domain),
|
|
264
|
+
cube: metric.cube,
|
|
265
|
+
table: metric.table,
|
|
266
|
+
sql: metric.sql,
|
|
267
|
+
type: metric.type,
|
|
268
|
+
tags: metric.tags ?? [],
|
|
269
|
+
owner: metric.owner ?? null,
|
|
270
|
+
source: metric.source ?? manifestObject?.source ?? null,
|
|
271
|
+
filePath: manifestObject?.filePath ?? null,
|
|
272
|
+
importedAt,
|
|
273
|
+
};
|
|
274
|
+
}
|
|
275
|
+
if (kind === 'dimension') {
|
|
276
|
+
const dimension = layer.getDimension(name);
|
|
277
|
+
if (!dimension)
|
|
278
|
+
return null;
|
|
279
|
+
return {
|
|
280
|
+
id,
|
|
281
|
+
kind: 'dimension',
|
|
282
|
+
name: dimension.name,
|
|
283
|
+
label: dimension.label,
|
|
284
|
+
description: dimension.description,
|
|
285
|
+
domain: normalizeDomain(dimension.domain),
|
|
286
|
+
cube: dimension.cube,
|
|
287
|
+
table: dimension.table,
|
|
288
|
+
sql: dimension.sql,
|
|
289
|
+
type: dimension.type,
|
|
290
|
+
tags: dimension.tags ?? [],
|
|
291
|
+
owner: dimension.owner ?? null,
|
|
292
|
+
source: dimension.source ?? manifestObject?.source ?? null,
|
|
293
|
+
filePath: manifestObject?.filePath ?? null,
|
|
294
|
+
importedAt,
|
|
295
|
+
};
|
|
296
|
+
}
|
|
297
|
+
if (kind === 'hierarchy') {
|
|
298
|
+
const hierarchy = layer.getHierarchy(name);
|
|
299
|
+
if (!hierarchy)
|
|
300
|
+
return null;
|
|
301
|
+
return {
|
|
302
|
+
id,
|
|
303
|
+
kind: 'hierarchy',
|
|
304
|
+
name: hierarchy.name,
|
|
305
|
+
label: hierarchy.label,
|
|
306
|
+
description: hierarchy.description,
|
|
307
|
+
domain: normalizeDomain(hierarchy.domain),
|
|
308
|
+
tags: hierarchy.tags ?? [],
|
|
309
|
+
owner: hierarchy.owner ?? null,
|
|
310
|
+
source: hierarchy.source ?? manifestObject?.source ?? null,
|
|
311
|
+
filePath: manifestObject?.filePath ?? null,
|
|
312
|
+
importedAt,
|
|
313
|
+
levels: hierarchy.levels,
|
|
314
|
+
};
|
|
315
|
+
}
|
|
316
|
+
if (kind === 'segment') {
|
|
317
|
+
const segment = layer.getSegment(name);
|
|
318
|
+
if (!segment)
|
|
319
|
+
return null;
|
|
320
|
+
return {
|
|
321
|
+
id,
|
|
322
|
+
kind: 'segment',
|
|
323
|
+
name: segment.name,
|
|
324
|
+
label: segment.label,
|
|
325
|
+
description: segment.description,
|
|
326
|
+
domain: normalizeDomain(segment.domain),
|
|
327
|
+
cube: segment.cube,
|
|
328
|
+
sql: segment.sql,
|
|
329
|
+
tags: segment.tags ?? [],
|
|
330
|
+
owner: segment.owner ?? null,
|
|
331
|
+
source: segment.source ?? manifestObject?.source ?? null,
|
|
332
|
+
filePath: manifestObject?.filePath ?? null,
|
|
333
|
+
importedAt,
|
|
334
|
+
};
|
|
335
|
+
}
|
|
336
|
+
if (kind === 'pre_aggregation') {
|
|
337
|
+
const preAggregation = layer.getPreAggregation(name);
|
|
338
|
+
if (!preAggregation)
|
|
339
|
+
return null;
|
|
340
|
+
return {
|
|
341
|
+
id,
|
|
342
|
+
kind: 'pre_aggregation',
|
|
343
|
+
name: preAggregation.name,
|
|
344
|
+
label: preAggregation.label,
|
|
345
|
+
description: preAggregation.description,
|
|
346
|
+
domain: normalizeDomain(preAggregation.domain),
|
|
347
|
+
cube: preAggregation.cube,
|
|
348
|
+
sql: preAggregation.sql,
|
|
349
|
+
tags: preAggregation.tags ?? [],
|
|
350
|
+
owner: preAggregation.owner ?? null,
|
|
351
|
+
source: preAggregation.source ?? manifestObject?.source ?? null,
|
|
352
|
+
filePath: manifestObject?.filePath ?? null,
|
|
353
|
+
importedAt,
|
|
354
|
+
measures: preAggregation.measures,
|
|
355
|
+
dimensions: preAggregation.dimensions,
|
|
356
|
+
timeDimension: preAggregation.timeDimension,
|
|
357
|
+
granularity: preAggregation.granularity,
|
|
358
|
+
refreshKey: preAggregation.refreshKey,
|
|
359
|
+
};
|
|
360
|
+
}
|
|
361
|
+
return null;
|
|
362
|
+
}
|
|
363
|
+
/**
|
|
364
|
+
* Preview a semantic import without writing any files.
|
|
365
|
+
* Returns counts, domains, warnings, and object summaries.
|
|
366
|
+
*/
|
|
367
|
+
export async function previewSemanticImport(opts) {
|
|
368
|
+
const targetProjectRoot = resolve(opts.targetProjectRoot);
|
|
369
|
+
const source = resolveImportSource(targetProjectRoot, opts.sourceConfig);
|
|
370
|
+
const layer = await loadLayerForImport(opts.provider, source.localPath, source.config, opts.executeQuery);
|
|
371
|
+
const warnings = [...source.warnings, ...collectImportWarnings(opts.provider, layer)];
|
|
372
|
+
const objects = collectObjects(layer);
|
|
373
|
+
const summaries = objects.map((obj) => ({
|
|
374
|
+
kind: obj.kind,
|
|
375
|
+
name: obj.name,
|
|
376
|
+
label: obj.label,
|
|
377
|
+
domain: normalizeDomain(obj.domain),
|
|
378
|
+
}));
|
|
379
|
+
const domains = [...new Set(summaries.map((o) => o.domain))].sort();
|
|
380
|
+
return {
|
|
381
|
+
provider: opts.provider,
|
|
382
|
+
counts: countObjects(summaries.map((o) => ({ ...o, id: objectId(o.kind, o.name), filePath: '' }))),
|
|
383
|
+
domains,
|
|
384
|
+
warnings,
|
|
385
|
+
objects: summaries,
|
|
386
|
+
};
|
|
387
|
+
}
|
|
388
|
+
/**
|
|
389
|
+
* Compute the diff between the current manifest and a fresh import, without applying changes.
|
|
390
|
+
*/
|
|
391
|
+
export async function computeSyncDiff(opts) {
|
|
392
|
+
const manifest = loadSemanticImportManifest(opts.targetProjectRoot);
|
|
393
|
+
if (!manifest) {
|
|
394
|
+
throw new Error('No semantic import manifest found. Run `dql semantic import <provider>` first.');
|
|
395
|
+
}
|
|
396
|
+
const sourceConfig = {
|
|
397
|
+
provider: manifest.provider,
|
|
398
|
+
projectPath: manifest.source.projectPath,
|
|
399
|
+
repoUrl: manifest.source.repoUrl,
|
|
400
|
+
branch: manifest.source.branch,
|
|
401
|
+
subPath: manifest.source.subPath,
|
|
402
|
+
connection: manifest.source.connection,
|
|
403
|
+
};
|
|
404
|
+
const targetProjectRoot = resolve(opts.targetProjectRoot);
|
|
405
|
+
const source = resolveImportSource(targetProjectRoot, sourceConfig);
|
|
406
|
+
const layer = await loadLayerForImport(manifest.provider, source.localPath, source.config, opts.executeQuery);
|
|
407
|
+
const newObjects = collectObjects(layer);
|
|
408
|
+
const newById = new Map(newObjects.map((obj) => [objectId(obj.kind, obj.name), obj]));
|
|
409
|
+
const oldById = new Map(manifest.objects.map((obj) => [obj.id, obj]));
|
|
410
|
+
const added = [];
|
|
411
|
+
const removed = [];
|
|
412
|
+
const changed = [];
|
|
413
|
+
let unchanged = 0;
|
|
414
|
+
for (const [id, obj] of newById) {
|
|
415
|
+
const old = oldById.get(id);
|
|
416
|
+
if (!old) {
|
|
417
|
+
added.push({ kind: obj.kind, name: obj.name, label: obj.label, domain: normalizeDomain(obj.domain) });
|
|
418
|
+
}
|
|
419
|
+
else {
|
|
420
|
+
// Compare domain and label to detect changes
|
|
421
|
+
const oldDomain = old.domain;
|
|
422
|
+
const newDomain = normalizeDomain(obj.domain);
|
|
423
|
+
if (old.label !== obj.label || oldDomain !== newDomain) {
|
|
424
|
+
changed.push({ kind: obj.kind, name: obj.name, label: obj.label, domain: newDomain });
|
|
425
|
+
}
|
|
426
|
+
else {
|
|
427
|
+
unchanged++;
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
for (const [id, old] of oldById) {
|
|
432
|
+
if (!newById.has(id)) {
|
|
433
|
+
removed.push({ kind: old.kind, name: old.name, label: old.label, domain: old.domain });
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
return { added, removed, changed, unchanged };
|
|
437
|
+
}
|
|
438
|
+
function resolveImportSource(targetProjectRoot, sourceConfig) {
|
|
439
|
+
const projectRoot = resolve(targetProjectRoot);
|
|
440
|
+
if (sourceConfig.provider === 'snowflake') {
|
|
441
|
+
return {
|
|
442
|
+
localPath: projectRoot,
|
|
443
|
+
config: sourceConfig,
|
|
444
|
+
warnings: [],
|
|
445
|
+
};
|
|
446
|
+
}
|
|
447
|
+
if (sourceConfig.projectPath && !sourceConfig.repoUrl && sourceConfig.source !== 'github' && sourceConfig.source !== 'gitlab') {
|
|
448
|
+
return {
|
|
449
|
+
localPath: resolve(projectRoot, sourceConfig.projectPath),
|
|
450
|
+
config: { ...sourceConfig, projectPath: undefined },
|
|
451
|
+
warnings: [],
|
|
452
|
+
};
|
|
453
|
+
}
|
|
454
|
+
const resolved = resolveRepoSource(sourceConfig, projectRoot);
|
|
455
|
+
return {
|
|
456
|
+
localPath: resolved.localPath,
|
|
457
|
+
config: { ...sourceConfig, projectPath: undefined, source: 'local' },
|
|
458
|
+
warnings: resolved.warnings,
|
|
459
|
+
};
|
|
460
|
+
}
|
|
461
|
+
async function loadLayerForImport(provider, sourceRoot, config, executeQuery) {
|
|
462
|
+
if (provider === 'dbt') {
|
|
463
|
+
return new DbtProvider().load({ ...config, provider }, sourceRoot);
|
|
464
|
+
}
|
|
465
|
+
if (provider === 'cubejs') {
|
|
466
|
+
return new CubejsProvider().load({ ...config, provider }, sourceRoot);
|
|
467
|
+
}
|
|
468
|
+
if (!executeQuery) {
|
|
469
|
+
throw new Error('Snowflake semantic import requires an active query executor.');
|
|
470
|
+
}
|
|
471
|
+
return new SnowflakeSemanticProvider(executeQuery).loadAsync({ ...config, provider }, sourceRoot);
|
|
472
|
+
}
|
|
473
|
+
function collectObjects(layer) {
|
|
474
|
+
return [
|
|
475
|
+
...layer.listCubes().map((cube) => ({ ...cube, kind: 'cube' })),
|
|
476
|
+
...layer.listMetrics().map((metric) => ({ ...metric, kind: 'metric' })),
|
|
477
|
+
...layer.listDimensions().map((dimension) => ({ ...dimension, kind: 'dimension' })),
|
|
478
|
+
...layer.listHierarchies().map((hierarchy) => ({ ...hierarchy, kind: 'hierarchy' })),
|
|
479
|
+
...layer.listSegments().map((segment) => ({ ...segment, kind: 'segment' })),
|
|
480
|
+
...layer.listPreAggregations().map((preAggregation) => ({ ...preAggregation, kind: 'pre_aggregation' })),
|
|
481
|
+
];
|
|
482
|
+
}
|
|
483
|
+
function buildSemanticFilePath(kind, domain, name) {
|
|
484
|
+
const folder = kind === 'pre_aggregation' ? 'pre_aggregations' : `${kind}s`.replace('hierarchys', 'hierarchies');
|
|
485
|
+
return join('semantic-layer', folder, slugifyPathSegment(domain), `${slugifyPathSegment(name)}.yaml`);
|
|
486
|
+
}
|
|
487
|
+
function serializeSemanticObject(object) {
|
|
488
|
+
const lines = [
|
|
489
|
+
`name: ${yamlScalar(object.name)}`,
|
|
490
|
+
`label: ${yamlScalar(object.label)}`,
|
|
491
|
+
`description: ${yamlScalar(object.description)}`,
|
|
492
|
+
`domain: ${yamlScalar(object.domain || normalizeDomain(undefined))}`,
|
|
493
|
+
];
|
|
494
|
+
if ('table' in object && object.table)
|
|
495
|
+
lines.push(`table: ${yamlScalar(object.table)}`);
|
|
496
|
+
if ('cube' in object && object.cube)
|
|
497
|
+
lines.push(`cube: ${yamlScalar(object.cube)}`);
|
|
498
|
+
if ('sql' in object && typeof object.sql === 'string')
|
|
499
|
+
lines.push(`sql: ${yamlBlockScalar(object.sql)}`);
|
|
500
|
+
if ('type' in object && typeof object.type === 'string')
|
|
501
|
+
lines.push(`type: ${yamlScalar(object.type)}`);
|
|
502
|
+
if ('aggregation' in object && object.aggregation)
|
|
503
|
+
lines.push(`aggregation: ${yamlScalar(object.aggregation)}`);
|
|
504
|
+
if ('owner' in object && object.owner)
|
|
505
|
+
lines.push(`owner: ${yamlScalar(object.owner)}`);
|
|
506
|
+
if ('tags' in object && object.tags && object.tags.length > 0) {
|
|
507
|
+
lines.push('tags:');
|
|
508
|
+
for (const tag of object.tags)
|
|
509
|
+
lines.push(` - ${yamlScalar(tag)}`);
|
|
510
|
+
}
|
|
511
|
+
if (object.source) {
|
|
512
|
+
lines.push('source:');
|
|
513
|
+
lines.push(` provider: ${yamlScalar(object.source.provider)}`);
|
|
514
|
+
lines.push(` objectType: ${yamlScalar(object.source.objectType)}`);
|
|
515
|
+
lines.push(` objectId: ${yamlScalar(object.source.objectId)}`);
|
|
516
|
+
if (object.source.objectName)
|
|
517
|
+
lines.push(` objectName: ${yamlScalar(object.source.objectName)}`);
|
|
518
|
+
if (object.source.importedAt)
|
|
519
|
+
lines.push(` importedAt: ${yamlScalar(object.source.importedAt)}`);
|
|
520
|
+
if (object.source.extra && Object.keys(object.source.extra).length > 0) {
|
|
521
|
+
lines.push(' extra:');
|
|
522
|
+
for (const [key, value] of Object.entries(object.source.extra)) {
|
|
523
|
+
lines.push(` ${key}: ${yamlScalar(String(value))}`);
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
}
|
|
527
|
+
if (object.kind === 'hierarchy') {
|
|
528
|
+
lines.push('levels:');
|
|
529
|
+
for (const level of object.levels) {
|
|
530
|
+
lines.push(` - name: ${yamlScalar(level.name)}`);
|
|
531
|
+
lines.push(` label: ${yamlScalar(level.label)}`);
|
|
532
|
+
lines.push(` description: ${yamlScalar(level.description)}`);
|
|
533
|
+
lines.push(` dimension: ${yamlScalar(level.dimension)}`);
|
|
534
|
+
lines.push(` order: ${level.order}`);
|
|
535
|
+
}
|
|
536
|
+
}
|
|
537
|
+
if (object.kind === 'cube') {
|
|
538
|
+
lines.push('measures:');
|
|
539
|
+
for (const measure of object.measures) {
|
|
540
|
+
lines.push(` - name: ${yamlScalar(measure.name)}`);
|
|
541
|
+
lines.push(` label: ${yamlScalar(measure.label)}`);
|
|
542
|
+
lines.push(` description: ${yamlScalar(measure.description)}`);
|
|
543
|
+
lines.push(` sql: ${yamlBlockScalar(measure.sql, 4)}`);
|
|
544
|
+
lines.push(` type: ${yamlScalar(measure.type)}`);
|
|
545
|
+
if (measure.aggregation)
|
|
546
|
+
lines.push(` aggregation: ${yamlScalar(measure.aggregation)}`);
|
|
547
|
+
}
|
|
548
|
+
lines.push('dimensions:');
|
|
549
|
+
for (const dimension of object.dimensions) {
|
|
550
|
+
lines.push(` - name: ${yamlScalar(dimension.name)}`);
|
|
551
|
+
lines.push(` label: ${yamlScalar(dimension.label)}`);
|
|
552
|
+
lines.push(` description: ${yamlScalar(dimension.description)}`);
|
|
553
|
+
lines.push(` sql: ${yamlBlockScalar(dimension.sql, 4)}`);
|
|
554
|
+
lines.push(` type: ${yamlScalar(dimension.type)}`);
|
|
555
|
+
}
|
|
556
|
+
if (object.timeDimensions.length > 0) {
|
|
557
|
+
lines.push('time_dimensions:');
|
|
558
|
+
for (const dimension of object.timeDimensions) {
|
|
559
|
+
lines.push(` - name: ${yamlScalar(dimension.name)}`);
|
|
560
|
+
lines.push(` label: ${yamlScalar(dimension.label)}`);
|
|
561
|
+
lines.push(` description: ${yamlScalar(dimension.description)}`);
|
|
562
|
+
lines.push(` sql: ${yamlBlockScalar(dimension.sql, 4)}`);
|
|
563
|
+
lines.push(' granularities:');
|
|
564
|
+
for (const granularity of dimension.granularities) {
|
|
565
|
+
lines.push(` - ${yamlScalar(granularity)}`);
|
|
566
|
+
}
|
|
567
|
+
if (dimension.primaryTime)
|
|
568
|
+
lines.push(' primary_time: true');
|
|
569
|
+
}
|
|
570
|
+
}
|
|
571
|
+
if (object.joins.length > 0) {
|
|
572
|
+
lines.push('joins:');
|
|
573
|
+
for (const joinDef of object.joins) {
|
|
574
|
+
lines.push(` - name: ${yamlScalar(joinDef.name)}`);
|
|
575
|
+
lines.push(` right: ${yamlScalar(joinDef.right)}`);
|
|
576
|
+
lines.push(` type: ${yamlScalar(joinDef.type)}`);
|
|
577
|
+
lines.push(` sql: ${yamlBlockScalar(joinDef.sql, 4)}`);
|
|
578
|
+
}
|
|
579
|
+
}
|
|
580
|
+
if (object.segments.length > 0) {
|
|
581
|
+
lines.push('segments:');
|
|
582
|
+
for (const segment of object.segments) {
|
|
583
|
+
lines.push(` - name: ${yamlScalar(segment.name)}`);
|
|
584
|
+
lines.push(` label: ${yamlScalar(segment.label)}`);
|
|
585
|
+
lines.push(` description: ${yamlScalar(segment.description)}`);
|
|
586
|
+
lines.push(` sql: ${yamlBlockScalar(segment.sql, 4)}`);
|
|
587
|
+
}
|
|
588
|
+
}
|
|
589
|
+
if (object.preAggregations.length > 0) {
|
|
590
|
+
lines.push('pre_aggregations:');
|
|
591
|
+
for (const preAggregation of object.preAggregations) {
|
|
592
|
+
lines.push(` - name: ${yamlScalar(preAggregation.name)}`);
|
|
593
|
+
lines.push(` label: ${yamlScalar(preAggregation.label)}`);
|
|
594
|
+
lines.push(` description: ${yamlScalar(preAggregation.description)}`);
|
|
595
|
+
if (preAggregation.measures?.length) {
|
|
596
|
+
lines.push(' measures:');
|
|
597
|
+
for (const measure of preAggregation.measures)
|
|
598
|
+
lines.push(` - ${yamlScalar(measure)}`);
|
|
599
|
+
}
|
|
600
|
+
if (preAggregation.dimensions?.length) {
|
|
601
|
+
lines.push(' dimensions:');
|
|
602
|
+
for (const dimension of preAggregation.dimensions)
|
|
603
|
+
lines.push(` - ${yamlScalar(dimension)}`);
|
|
604
|
+
}
|
|
605
|
+
if (preAggregation.timeDimension)
|
|
606
|
+
lines.push(` timeDimension: ${yamlScalar(preAggregation.timeDimension)}`);
|
|
607
|
+
if (preAggregation.granularity)
|
|
608
|
+
lines.push(` granularity: ${yamlScalar(preAggregation.granularity)}`);
|
|
609
|
+
if (preAggregation.refreshKey)
|
|
610
|
+
lines.push(` refreshKey: ${yamlScalar(preAggregation.refreshKey)}`);
|
|
611
|
+
}
|
|
612
|
+
}
|
|
613
|
+
}
|
|
614
|
+
if (object.kind === 'pre_aggregation') {
|
|
615
|
+
if (object.measures?.length) {
|
|
616
|
+
lines.push('measures:');
|
|
617
|
+
for (const measure of object.measures)
|
|
618
|
+
lines.push(` - ${yamlScalar(measure)}`);
|
|
619
|
+
}
|
|
620
|
+
if (object.dimensions?.length) {
|
|
621
|
+
lines.push('dimensions:');
|
|
622
|
+
for (const dimension of object.dimensions)
|
|
623
|
+
lines.push(` - ${yamlScalar(dimension)}`);
|
|
624
|
+
}
|
|
625
|
+
if (object.timeDimension)
|
|
626
|
+
lines.push(`timeDimension: ${yamlScalar(object.timeDimension)}`);
|
|
627
|
+
if (object.granularity)
|
|
628
|
+
lines.push(`granularity: ${yamlScalar(object.granularity)}`);
|
|
629
|
+
if (object.refreshKey)
|
|
630
|
+
lines.push(`refreshKey: ${yamlScalar(object.refreshKey)}`);
|
|
631
|
+
}
|
|
632
|
+
return lines.join('\n') + '\n';
|
|
633
|
+
}
|
|
634
|
+
function applyCanonicalSemanticConfig(projectRoot) {
|
|
635
|
+
const configPath = join(projectRoot, 'dql.config.json');
|
|
636
|
+
const raw = existsSync(configPath)
|
|
637
|
+
? JSON.parse(readFileSync(configPath, 'utf-8'))
|
|
638
|
+
: {};
|
|
639
|
+
raw.semanticLayer = {
|
|
640
|
+
provider: 'dql',
|
|
641
|
+
path: './semantic-layer',
|
|
642
|
+
};
|
|
643
|
+
writeFileSync(configPath, JSON.stringify(raw, null, 2) + '\n', 'utf-8');
|
|
644
|
+
}
|
|
645
|
+
function collectImportWarnings(provider, layer) {
|
|
646
|
+
const warnings = [];
|
|
647
|
+
if (provider === 'dbt') {
|
|
648
|
+
if (layer.listHierarchies().length === 0) {
|
|
649
|
+
warnings.push('No dbt hierarchies were imported; dbt semantic models were normalized into cubes, measures, dimensions, and joins.');
|
|
650
|
+
}
|
|
651
|
+
}
|
|
652
|
+
if (provider === 'snowflake' && layer.listCubes().length === 0) {
|
|
653
|
+
warnings.push('Snowflake semantic import returned no semantic views.');
|
|
654
|
+
}
|
|
655
|
+
return warnings;
|
|
656
|
+
}
|
|
657
|
+
function countObjects(objects) {
|
|
658
|
+
return {
|
|
659
|
+
cube: objects.filter((object) => object.kind === 'cube').length,
|
|
660
|
+
metric: objects.filter((object) => object.kind === 'metric').length,
|
|
661
|
+
dimension: objects.filter((object) => object.kind === 'dimension').length,
|
|
662
|
+
hierarchy: objects.filter((object) => object.kind === 'hierarchy').length,
|
|
663
|
+
segment: objects.filter((object) => object.kind === 'segment').length,
|
|
664
|
+
pre_aggregation: objects.filter((object) => object.kind === 'pre_aggregation').length,
|
|
665
|
+
};
|
|
666
|
+
}
|
|
667
|
+
function normalizeDomain(domain) {
|
|
668
|
+
return domain && domain.trim().length > 0 ? domain.trim() : 'uncategorized';
|
|
669
|
+
}
|
|
670
|
+
function slugifyPathSegment(value) {
|
|
671
|
+
return normalizeDomain(value)
|
|
672
|
+
.toLowerCase()
|
|
673
|
+
.replace(/[^a-z0-9/_-]+/g, '-')
|
|
674
|
+
.replace(/\/+/g, '/')
|
|
675
|
+
.replace(/^-+|-+$/g, '');
|
|
676
|
+
}
|
|
677
|
+
function objectId(kind, name) {
|
|
678
|
+
return `${kind}:${name}`;
|
|
679
|
+
}
|
|
680
|
+
function yamlScalar(value) {
|
|
681
|
+
if (/^[a-zA-Z0-9_.:/-]+$/.test(value))
|
|
682
|
+
return value;
|
|
683
|
+
return JSON.stringify(value);
|
|
684
|
+
}
|
|
685
|
+
function yamlBlockScalar(value, indent = 2) {
|
|
686
|
+
const indentText = ' '.repeat(indent);
|
|
687
|
+
if (!value.includes('\n'))
|
|
688
|
+
return yamlScalar(value);
|
|
689
|
+
return `|\n${value.split('\n').map((line) => `${indentText}${line}`).join('\n')}`;
|
|
690
|
+
}
|
|
691
|
+
function buildGroupNode(kind, label, children) {
|
|
692
|
+
if (children.length === 0)
|
|
693
|
+
return null;
|
|
694
|
+
return {
|
|
695
|
+
id: `group:${kind}:${label.toLowerCase()}`,
|
|
696
|
+
label,
|
|
697
|
+
kind: 'group',
|
|
698
|
+
count: children.length,
|
|
699
|
+
meta: {
|
|
700
|
+
objectKind: kind,
|
|
701
|
+
},
|
|
702
|
+
children,
|
|
703
|
+
};
|
|
704
|
+
}
|
|
705
|
+
function toLeaf(kind, name, label, meta) {
|
|
706
|
+
return {
|
|
707
|
+
id: objectId(kind, name),
|
|
708
|
+
label,
|
|
709
|
+
kind,
|
|
710
|
+
meta,
|
|
711
|
+
};
|
|
712
|
+
}
|
|
713
|
+
//# sourceMappingURL=semantic-import.js.map
|