@duckcodeailabs/dql-cli 1.4.3 → 1.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/apps-api.d.ts +77 -0
- package/apps-api.d.ts.map +1 -0
- package/apps-api.js +612 -0
- package/apps-api.js.map +1 -0
- package/apps-api.test.d.ts +2 -0
- package/apps-api.test.d.ts.map +1 -0
- package/apps-api.test.js +111 -0
- package/apps-api.test.js.map +1 -0
- package/args.d.ts +30 -0
- package/args.d.ts.map +1 -0
- package/args.js +105 -0
- package/args.js.map +1 -0
- package/args.test.d.ts +2 -0
- package/args.test.d.ts.map +1 -0
- package/args.test.js +33 -0
- package/args.test.js.map +1 -0
- package/assets/dql-notebook/assets/codemirror-DJYUkPr1.js +11 -0
- package/assets/dql-notebook/assets/index-DUTeFz5j.js +858 -0
- package/assets/dql-notebook/assets/index-DrhoZmtv.css +1 -0
- package/assets/dql-notebook/assets/react-CRB3T2We.js +32 -0
- package/assets/dql-notebook/index.html +18 -0
- package/assets/notebook-browser/app.js +548 -0
- package/assets/notebook-browser/index.html +83 -0
- package/assets/notebook-browser/styles.css +336 -0
- package/block-templates.d.ts +8 -0
- package/block-templates.d.ts.map +1 -0
- package/block-templates.js +60 -0
- package/block-templates.js.map +1 -0
- package/commands/agent.d.ts +19 -0
- package/commands/agent.d.ts.map +1 -0
- package/commands/agent.js +165 -0
- package/commands/agent.js.map +1 -0
- package/commands/app.d.ts +32 -0
- package/commands/app.d.ts.map +1 -0
- package/commands/app.js +307 -0
- package/commands/app.js.map +1 -0
- package/commands/build.d.ts +3 -0
- package/commands/build.d.ts.map +1 -0
- package/commands/build.js +69 -0
- package/commands/build.js.map +1 -0
- package/commands/build.test.d.ts +2 -0
- package/commands/build.test.d.ts.map +1 -0
- package/commands/build.test.js +44 -0
- package/commands/build.test.js.map +1 -0
- package/commands/certify.d.ts +3 -0
- package/commands/certify.d.ts.map +1 -0
- package/commands/certify.js +228 -0
- package/commands/certify.js.map +1 -0
- package/commands/compile.d.ts +21 -0
- package/commands/compile.d.ts.map +1 -0
- package/commands/compile.js +198 -0
- package/commands/compile.js.map +1 -0
- package/commands/compile.test.d.ts +2 -0
- package/commands/compile.test.d.ts.map +1 -0
- package/commands/compile.test.js +115 -0
- package/commands/compile.test.js.map +1 -0
- package/commands/diff.d.ts +3 -0
- package/commands/diff.d.ts.map +1 -0
- package/commands/diff.js +52 -0
- package/commands/diff.js.map +1 -0
- package/commands/doctor.d.ts +3 -0
- package/commands/doctor.d.ts.map +1 -0
- package/commands/doctor.js +191 -0
- package/commands/doctor.js.map +1 -0
- package/commands/doctor.test.d.ts +2 -0
- package/commands/doctor.test.d.ts.map +1 -0
- package/commands/doctor.test.js +43 -0
- package/commands/doctor.test.js.map +1 -0
- package/commands/fmt.d.ts +3 -0
- package/commands/fmt.d.ts.map +1 -0
- package/commands/fmt.js +53 -0
- package/commands/fmt.js.map +1 -0
- package/commands/info.d.ts +3 -0
- package/commands/info.d.ts.map +1 -0
- package/commands/info.js +56 -0
- package/commands/info.js.map +1 -0
- package/commands/init.d.ts +3 -0
- package/commands/init.d.ts.map +1 -0
- package/commands/init.js +250 -0
- package/commands/init.js.map +1 -0
- package/commands/init.test.d.ts +2 -0
- package/commands/init.test.d.ts.map +1 -0
- package/commands/init.test.js +118 -0
- package/commands/init.test.js.map +1 -0
- package/commands/lineage.d.ts +24 -0
- package/commands/lineage.d.ts.map +1 -0
- package/commands/lineage.js +634 -0
- package/commands/lineage.js.map +1 -0
- package/commands/mcp.d.ts +7 -0
- package/commands/mcp.d.ts.map +1 -0
- package/commands/mcp.js +16 -0
- package/commands/mcp.js.map +1 -0
- package/commands/migrate.d.ts +12 -0
- package/commands/migrate.d.ts.map +1 -0
- package/commands/migrate.js +192 -0
- package/commands/migrate.js.map +1 -0
- package/commands/new.d.ts +3 -0
- package/commands/new.d.ts.map +1 -0
- package/commands/new.js +490 -0
- package/commands/new.js.map +1 -0
- package/commands/new.test.d.ts +2 -0
- package/commands/new.test.d.ts.map +1 -0
- package/commands/new.test.js +191 -0
- package/commands/new.test.js.map +1 -0
- package/commands/notebook.d.ts +3 -0
- package/commands/notebook.d.ts.map +1 -0
- package/commands/notebook.js +46 -0
- package/commands/notebook.js.map +1 -0
- package/commands/parse.d.ts +3 -0
- package/commands/parse.d.ts.map +1 -0
- package/commands/parse.js +63 -0
- package/commands/parse.js.map +1 -0
- package/commands/preview.d.ts +3 -0
- package/commands/preview.d.ts.map +1 -0
- package/commands/preview.js +42 -0
- package/commands/preview.js.map +1 -0
- package/commands/schedule.d.ts +3 -0
- package/commands/schedule.d.ts.map +1 -0
- package/commands/schedule.js +215 -0
- package/commands/schedule.js.map +1 -0
- package/commands/semantic.d.ts +12 -0
- package/commands/semantic.d.ts.map +1 -0
- package/commands/semantic.js +356 -0
- package/commands/semantic.js.map +1 -0
- package/commands/serve.d.ts +3 -0
- package/commands/serve.d.ts.map +1 -0
- package/commands/serve.js +30 -0
- package/commands/serve.js.map +1 -0
- package/commands/slack.d.ts +13 -0
- package/commands/slack.d.ts.map +1 -0
- package/commands/slack.js +53 -0
- package/commands/slack.js.map +1 -0
- package/commands/sync.d.ts +3 -0
- package/commands/sync.d.ts.map +1 -0
- package/commands/sync.js +192 -0
- package/commands/sync.js.map +1 -0
- package/commands/sync.test.d.ts +2 -0
- package/commands/sync.test.d.ts.map +1 -0
- package/commands/sync.test.js +147 -0
- package/commands/sync.test.js.map +1 -0
- package/commands/test.d.ts +3 -0
- package/commands/test.d.ts.map +1 -0
- package/commands/test.js +167 -0
- package/commands/test.js.map +1 -0
- package/commands/validate.d.ts +3 -0
- package/commands/validate.d.ts.map +1 -0
- package/commands/validate.js +116 -0
- package/commands/validate.js.map +1 -0
- package/commands/verify.d.ts +11 -0
- package/commands/verify.d.ts.map +1 -0
- package/commands/verify.js +74 -0
- package/commands/verify.js.map +1 -0
- package/digest.d.ts +10 -0
- package/digest.d.ts.map +1 -0
- package/digest.js +83 -0
- package/digest.js.map +1 -0
- package/git-service.d.ts +17 -0
- package/git-service.d.ts.map +1 -0
- package/git-service.js +54 -0
- package/git-service.js.map +1 -0
- package/governance-runtime.d.ts +15 -0
- package/governance-runtime.d.ts.map +1 -0
- package/governance-runtime.js +50 -0
- package/governance-runtime.js.map +1 -0
- package/index.d.ts +3 -0
- package/index.d.ts.map +1 -0
- package/index.js.map +1 -0
- package/llm/index.d.ts +4 -0
- package/llm/index.d.ts.map +1 -0
- package/llm/index.js +19 -0
- package/llm/index.js.map +1 -0
- package/llm/providers/claude-agent-sdk.d.ts +3 -0
- package/llm/providers/claude-agent-sdk.d.ts.map +1 -0
- package/llm/providers/claude-agent-sdk.js +174 -0
- package/llm/providers/claude-agent-sdk.js.map +1 -0
- package/llm/providers/claude-code.d.ts +8 -0
- package/llm/providers/claude-code.d.ts.map +1 -0
- package/llm/providers/claude-code.js +171 -0
- package/llm/providers/claude-code.js.map +1 -0
- package/llm/providers/dql-agent-provider.d.ts +5 -0
- package/llm/providers/dql-agent-provider.d.ts.map +1 -0
- package/llm/providers/dql-agent-provider.js +99 -0
- package/llm/providers/dql-agent-provider.js.map +1 -0
- package/llm/tools.d.ts +9 -0
- package/llm/tools.d.ts.map +1 -0
- package/llm/tools.js +112 -0
- package/llm/tools.js.map +1 -0
- package/llm/types.d.ts +70 -0
- package/llm/types.d.ts.map +1 -0
- package/llm/types.js +2 -0
- package/llm/types.js.map +1 -0
- package/local-runtime.d.ts +142 -0
- package/local-runtime.d.ts.map +1 -0
- package/local-runtime.js +4357 -0
- package/local-runtime.js.map +1 -0
- package/local-runtime.test.d.ts +2 -0
- package/local-runtime.test.d.ts.map +1 -0
- package/local-runtime.test.js +241 -0
- package/local-runtime.test.js.map +1 -0
- package/metricflow.d.ts +35 -0
- package/metricflow.d.ts.map +1 -0
- package/metricflow.js +122 -0
- package/metricflow.js.map +1 -0
- package/metricflow.test.d.ts +2 -0
- package/metricflow.test.d.ts.map +1 -0
- package/metricflow.test.js +54 -0
- package/metricflow.test.js.map +1 -0
- package/open-browser.d.ts +2 -0
- package/open-browser.d.ts.map +1 -0
- package/open-browser.js +29 -0
- package/open-browser.js.map +1 -0
- package/package.json +10 -13
- package/schedule/alerts.d.ts +5 -0
- package/schedule/alerts.d.ts.map +1 -0
- package/schedule/alerts.js +54 -0
- package/schedule/alerts.js.map +1 -0
- package/schedule/discovery.d.ts +4 -0
- package/schedule/discovery.d.ts.map +1 -0
- package/schedule/discovery.js +36 -0
- package/schedule/discovery.js.map +1 -0
- package/schedule/notifiers/email.d.ts +3 -0
- package/schedule/notifiers/email.d.ts.map +1 -0
- package/schedule/notifiers/email.js +76 -0
- package/schedule/notifiers/email.js.map +1 -0
- package/schedule/notifiers/file.d.ts +3 -0
- package/schedule/notifiers/file.d.ts.map +1 -0
- package/schedule/notifiers/file.js +50 -0
- package/schedule/notifiers/file.js.map +1 -0
- package/schedule/notifiers/index.d.ts +10 -0
- package/schedule/notifiers/index.d.ts.map +1 -0
- package/schedule/notifiers/index.js +33 -0
- package/schedule/notifiers/index.js.map +1 -0
- package/schedule/notifiers/slack.d.ts +3 -0
- package/schedule/notifiers/slack.d.ts.map +1 -0
- package/schedule/notifiers/slack.js +58 -0
- package/schedule/notifiers/slack.js.map +1 -0
- package/schedule/runner.d.ts +14 -0
- package/schedule/runner.d.ts.map +1 -0
- package/schedule/runner.js +221 -0
- package/schedule/runner.js.map +1 -0
- package/schedule/runs.d.ts +5 -0
- package/schedule/runs.d.ts.map +1 -0
- package/schedule/runs.js +41 -0
- package/schedule/runs.js.map +1 -0
- package/schedule/service.d.ts +13 -0
- package/schedule/service.d.ts.map +1 -0
- package/schedule/service.js +87 -0
- package/schedule/service.js.map +1 -0
- package/schedule/types.d.ts +70 -0
- package/schedule/types.d.ts.map +1 -0
- package/schedule/types.js +2 -0
- package/schedule/types.js.map +1 -0
- package/semantic-import.d.ts +135 -0
- package/semantic-import.d.ts.map +1 -0
- package/semantic-import.js +979 -0
- package/semantic-import.js.map +1 -0
- package/semantic-import.test.d.ts +2 -0
- package/semantic-import.test.d.ts.map +1 -0
- package/semantic-import.test.js +95 -0
- package/semantic-import.test.js.map +1 -0
package/local-runtime.js
ADDED
|
@@ -0,0 +1,4357 @@
|
|
|
1
|
+
import { execSync } from 'node:child_process';
|
|
2
|
+
import { createServer } from 'node:http';
|
|
3
|
+
import { existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, watch, writeFileSync } from 'node:fs';
|
|
4
|
+
import { dirname, extname, join, normalize, relative, resolve } from 'node:path';
|
|
5
|
+
import { buildExecutionPlan, createWelcomeNotebook, deserializeNotebook, getConnectorFormSchemas, hasSemanticRefs, resolveSemanticRefs, } from '@duckcodeailabs/dql-notebook';
|
|
6
|
+
import { loadSemanticLayerFromDir, resolveSemanticLayerAsync, Parser, buildLineageGraph, buildManifest, findAppDocuments, findDashboardsForApp, isBlockIdRef, loadAppDocument, loadDashboardDocument, analyzeImpact, buildTrustChain, detectDomainFlows, getDomainTrustOverview, queryLineage, queryCompleteLineagePaths, LineageGraph, canonicalize, canonicalizeNotebook, diffDQL, diffNotebook, } from '@duckcodeailabs/dql-core';
|
|
7
|
+
import { load as loadYaml } from 'js-yaml';
|
|
8
|
+
import { listBlockTemplates } from './block-templates.js';
|
|
9
|
+
import { getRunner as getLLMRunner } from './llm/index.js';
|
|
10
|
+
import { handleAppsApi } from './apps-api.js';
|
|
11
|
+
import { DQLAccessDeniedError, activePersonaAppId, assertAppAccess, loadRuntimeApp, runtimeVariables, } from './governance-runtime.js';
|
|
12
|
+
import { buildSemanticObjectDetail, buildSemanticTree, computeSyncDiff, loadSemanticImportManifest, performSemanticImport, previewSemanticImport, syncSemanticImport, } from './semantic-import.js';
|
|
13
|
+
import { MetricFlowUnavailableError, compileMetricFlowQuery, hasDbtSemanticManifest, } from './metricflow.js';
|
|
14
|
+
export async function startLocalServer(opts) {
|
|
15
|
+
const { rootDir, executor, connection: rawConnection, preferredPort, projectRoot = process.cwd() } = opts;
|
|
16
|
+
const bindHost = opts.host ?? process.env.DQL_HOST ?? '127.0.0.1';
|
|
17
|
+
let connection = normalizeProjectConnection(rawConnection, projectRoot);
|
|
18
|
+
let projectConfig = loadProjectConfig(projectRoot);
|
|
19
|
+
// Load semantic layer via provider system (dql native, dbt, cubejs, etc.)
|
|
20
|
+
let semanticLayer;
|
|
21
|
+
let semanticLayerErrors = [];
|
|
22
|
+
let semanticDetectedProvider;
|
|
23
|
+
const semanticLayerDir = join(projectRoot, 'semantic-layer');
|
|
24
|
+
let semanticImportManifest = loadSemanticImportManifest(projectRoot);
|
|
25
|
+
const userPrefsPath = join(projectRoot, '.dql-user-prefs.json');
|
|
26
|
+
const semanticConfig = projectConfig.semanticLayer;
|
|
27
|
+
let semanticLastSyncTime = null;
|
|
28
|
+
{
|
|
29
|
+
const executeQuery = semanticConfig?.provider === 'snowflake'
|
|
30
|
+
? async (sql) => { const r = await executor.executeQuery(sql, [], {}, connection); return { rows: r.rows }; }
|
|
31
|
+
: undefined;
|
|
32
|
+
const result = await resolveSemanticLayerAsync(semanticConfig, projectRoot, executeQuery);
|
|
33
|
+
semanticLayer = result.layer;
|
|
34
|
+
semanticLayerErrors = result.errors;
|
|
35
|
+
semanticDetectedProvider = result.detectedProvider;
|
|
36
|
+
semanticLastSyncTime = result.layer ? new Date().toISOString() : null;
|
|
37
|
+
semanticImportManifest = loadSemanticImportManifest(projectRoot);
|
|
38
|
+
// Legacy fallback if provider system returned nothing and no errors
|
|
39
|
+
if (!semanticLayer && semanticLayerErrors.length === 0 && existsSync(semanticLayerDir)) {
|
|
40
|
+
try {
|
|
41
|
+
semanticLayer = loadSemanticLayerFromDir(semanticLayerDir);
|
|
42
|
+
semanticLastSyncTime = new Date().toISOString();
|
|
43
|
+
}
|
|
44
|
+
catch { /* continue without */ }
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
// Auto-register data/ CSV and Parquet files as DuckDB views so semantic layer
|
|
48
|
+
// queries like `FROM orders` resolve without requiring read_csv_auto() in SQL.
|
|
49
|
+
if (connection.driver === 'file' || connection.driver === 'duckdb') {
|
|
50
|
+
const dataDir = projectConfig.dataDir
|
|
51
|
+
? resolve(projectRoot, projectConfig.dataDir)
|
|
52
|
+
: join(projectRoot, 'data');
|
|
53
|
+
if (existsSync(dataDir)) {
|
|
54
|
+
try {
|
|
55
|
+
const files = readdirSync(dataDir, { withFileTypes: true })
|
|
56
|
+
.filter((e) => e.isFile() && /\.(csv|parquet)$/i.test(e.name));
|
|
57
|
+
for (const file of files) {
|
|
58
|
+
const tableName = file.name.replace(/\.(csv|parquet)$/i, '');
|
|
59
|
+
const absPath = join(dataDir, file.name).replaceAll('\\', '/');
|
|
60
|
+
const reader = file.name.endsWith('.parquet') ? 'read_parquet' : 'read_csv_auto';
|
|
61
|
+
const ddl = `CREATE OR REPLACE VIEW "${tableName}" AS SELECT * FROM ${reader}('${absPath}')`;
|
|
62
|
+
try {
|
|
63
|
+
await executor.executeQuery(ddl, [], {}, connection);
|
|
64
|
+
}
|
|
65
|
+
catch { /* non-fatal */ }
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
catch { /* non-fatal */ }
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
// SSE clients for /api/watch hot-reload
|
|
72
|
+
const sseClients = new Set();
|
|
73
|
+
// Watch notebooks/, workbooks/, semantic-layer/, and data/ dirs for changes
|
|
74
|
+
if (projectRoot) {
|
|
75
|
+
for (const dir of ['notebooks', 'workbooks', 'blocks', 'dashboards', 'semantic-layer', 'data']) {
|
|
76
|
+
const watchDir = join(projectRoot, dir);
|
|
77
|
+
if (!existsSync(watchDir))
|
|
78
|
+
continue;
|
|
79
|
+
try {
|
|
80
|
+
watch(watchDir, { persistent: false }, (eventType, filename) => {
|
|
81
|
+
if (!filename)
|
|
82
|
+
return;
|
|
83
|
+
const path = `${dir}/${filename}`;
|
|
84
|
+
const payload = JSON.stringify({ type: eventType === 'rename' ? 'file-added' : 'file-changed', path });
|
|
85
|
+
for (const client of sseClients) {
|
|
86
|
+
try {
|
|
87
|
+
client.write(`event: change\ndata: ${payload}\n\n`);
|
|
88
|
+
}
|
|
89
|
+
catch {
|
|
90
|
+
sseClients.delete(client);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
// Hot-reload semantic layer on change and notify frontend
|
|
94
|
+
if (dir === 'semantic-layer') {
|
|
95
|
+
const executeQuery = semanticConfig?.provider === 'snowflake'
|
|
96
|
+
? async (sql) => { const r = await executor.executeQuery(sql, [], {}, connection); return { rows: r.rows }; }
|
|
97
|
+
: undefined;
|
|
98
|
+
resolveSemanticLayerAsync(semanticConfig, projectRoot, executeQuery).then((refreshed) => {
|
|
99
|
+
if (refreshed.layer) {
|
|
100
|
+
semanticLayer = refreshed.layer;
|
|
101
|
+
semanticLayerErrors = refreshed.errors;
|
|
102
|
+
semanticLastSyncTime = new Date().toISOString();
|
|
103
|
+
semanticImportManifest = loadSemanticImportManifest(projectRoot);
|
|
104
|
+
}
|
|
105
|
+
else if (refreshed.errors.length > 0) {
|
|
106
|
+
semanticLayerErrors = refreshed.errors;
|
|
107
|
+
}
|
|
108
|
+
// Notify all connected notebook clients to re-fetch the semantic layer
|
|
109
|
+
const reloadPayload = JSON.stringify({ type: 'semantic-reload' });
|
|
110
|
+
for (const client of sseClients) {
|
|
111
|
+
try {
|
|
112
|
+
client.write(`event: change\ndata: ${reloadPayload}\n\n`);
|
|
113
|
+
}
|
|
114
|
+
catch {
|
|
115
|
+
sseClients.delete(client);
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
}).catch(() => { });
|
|
119
|
+
}
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
catch { /* dir not watchable */ }
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
const server = createServer(async (req, res) => {
|
|
126
|
+
const requestUrl = req.url || '/';
|
|
127
|
+
const url = new URL(requestUrl, 'http://127.0.0.1');
|
|
128
|
+
const path = url.pathname || '/';
|
|
129
|
+
// CORS — needed for dql-notebook SPA dev mode
|
|
130
|
+
res.setHeader('Access-Control-Allow-Origin', '*');
|
|
131
|
+
res.setHeader('Access-Control-Allow-Methods', 'GET, POST, PUT, OPTIONS');
|
|
132
|
+
res.setHeader('Access-Control-Allow-Headers', 'Content-Type');
|
|
133
|
+
if (req.method === 'OPTIONS') {
|
|
134
|
+
res.writeHead(204);
|
|
135
|
+
res.end();
|
|
136
|
+
return;
|
|
137
|
+
}
|
|
138
|
+
if (req.method === 'GET' && path === '/api/health') {
|
|
139
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
140
|
+
res.end(serializeJSON({ status: 'ok' }));
|
|
141
|
+
return;
|
|
142
|
+
}
|
|
143
|
+
if (req.method === 'GET' && path === '/api/settings/env-status') {
|
|
144
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
145
|
+
res.end(serializeJSON({ groups: collectSettingsEnvStatus() }));
|
|
146
|
+
return;
|
|
147
|
+
}
|
|
148
|
+
const appDashRun = path.match(/^\/api\/apps\/([^/]+)\/dashboards\/([^/]+)\/run$/);
|
|
149
|
+
if (req.method === 'POST' && appDashRun) {
|
|
150
|
+
try {
|
|
151
|
+
const appId = decodeURIComponent(appDashRun[1]);
|
|
152
|
+
const dashboardId = decodeURIComponent(appDashRun[2]);
|
|
153
|
+
const body = await readJSON(req).catch(() => ({}));
|
|
154
|
+
const loaded = loadAppDashboard(projectRoot, appId, dashboardId);
|
|
155
|
+
if (!loaded) {
|
|
156
|
+
res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
157
|
+
res.end(serializeJSON({ error: `Dashboard "${dashboardId}" not found in app "${appId}"` }));
|
|
158
|
+
return;
|
|
159
|
+
}
|
|
160
|
+
const manifest = buildManifest({ projectRoot });
|
|
161
|
+
const variables = body.variables && typeof body.variables === 'object'
|
|
162
|
+
? body.variables
|
|
163
|
+
: {};
|
|
164
|
+
const tiles = [];
|
|
165
|
+
for (const item of loaded.dashboard.layout.items) {
|
|
166
|
+
const block = resolveDashboardItemBlock(item, manifest);
|
|
167
|
+
if (!block) {
|
|
168
|
+
tiles.push({
|
|
169
|
+
tileId: item.i,
|
|
170
|
+
status: 'unresolved',
|
|
171
|
+
blockRef: isBlockIdRef(item.block) ? item.block.blockId : item.block.ref,
|
|
172
|
+
error: 'Block reference could not be resolved',
|
|
173
|
+
});
|
|
174
|
+
continue;
|
|
175
|
+
}
|
|
176
|
+
try {
|
|
177
|
+
assertAppAccess({
|
|
178
|
+
app: loaded.app,
|
|
179
|
+
domain: block.domain ?? loaded.dashboard.metadata.domain ?? loaded.app.domain,
|
|
180
|
+
level: 'execute',
|
|
181
|
+
});
|
|
182
|
+
const absBlockPath = join(projectRoot, block.filePath);
|
|
183
|
+
const source = readFileSync(absBlockPath, 'utf-8');
|
|
184
|
+
const semanticCompose = semanticLayer
|
|
185
|
+
? composeSemanticBlockSql(source, semanticLayer, {
|
|
186
|
+
driver: connection.driver,
|
|
187
|
+
projectRoot,
|
|
188
|
+
projectConfig,
|
|
189
|
+
detectedProvider: semanticDetectedProvider,
|
|
190
|
+
})
|
|
191
|
+
: null;
|
|
192
|
+
const plan = buildExecutionPlan({ id: item.i, type: 'dql', source, title: item.title ?? block.name }, { semanticLayer, driver: connection.driver });
|
|
193
|
+
if (!plan && !semanticCompose?.sql) {
|
|
194
|
+
tiles.push({
|
|
195
|
+
tileId: item.i,
|
|
196
|
+
status: 'error',
|
|
197
|
+
blockId: block.name,
|
|
198
|
+
error: semanticCompose?.diagnostics.find((diagnostic) => diagnostic.severity === 'error')?.message ?? 'Block produced no executable plan',
|
|
199
|
+
});
|
|
200
|
+
continue;
|
|
201
|
+
}
|
|
202
|
+
const prepared = prepareLocalExecution(semanticCompose?.sql ?? plan.sql, isConnectionConfig(body.connection) ? body.connection : connection, projectRoot, projectConfig);
|
|
203
|
+
const result = await executor.executeQuery(prepared.sql, plan?.sqlParams ?? [], runtimeVariables({ ...(plan?.variables ?? {}), ...variables }), prepared.connection);
|
|
204
|
+
tiles.push({
|
|
205
|
+
tileId: item.i,
|
|
206
|
+
status: 'ok',
|
|
207
|
+
blockId: block.name,
|
|
208
|
+
blockPath: block.filePath,
|
|
209
|
+
certificationStatus: block.status ?? null,
|
|
210
|
+
title: item.title ?? block.name,
|
|
211
|
+
viz: item.viz,
|
|
212
|
+
chartConfig: plan?.chartConfig ?? { chart: item.viz.type },
|
|
213
|
+
result: normalizeQueryResult(result),
|
|
214
|
+
citation: {
|
|
215
|
+
kind: 'block',
|
|
216
|
+
name: block.name,
|
|
217
|
+
path: block.filePath,
|
|
218
|
+
},
|
|
219
|
+
});
|
|
220
|
+
}
|
|
221
|
+
catch (err) {
|
|
222
|
+
if (err instanceof DQLAccessDeniedError) {
|
|
223
|
+
tiles.push({
|
|
224
|
+
tileId: item.i,
|
|
225
|
+
status: 'unauthorized',
|
|
226
|
+
blockId: block.name,
|
|
227
|
+
error: err.message,
|
|
228
|
+
});
|
|
229
|
+
}
|
|
230
|
+
else {
|
|
231
|
+
tiles.push({
|
|
232
|
+
tileId: item.i,
|
|
233
|
+
status: 'error',
|
|
234
|
+
blockId: block.name,
|
|
235
|
+
error: err instanceof Error ? err.message : String(err),
|
|
236
|
+
});
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
241
|
+
res.end(serializeJSON({
|
|
242
|
+
appId,
|
|
243
|
+
dashboardId,
|
|
244
|
+
persona: activePersonaAppId() ? { appId: activePersonaAppId() } : null,
|
|
245
|
+
tiles,
|
|
246
|
+
}));
|
|
247
|
+
}
|
|
248
|
+
catch (err) {
|
|
249
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
250
|
+
res.end(serializeJSON({ error: err instanceof Error ? err.message : String(err) }));
|
|
251
|
+
}
|
|
252
|
+
return;
|
|
253
|
+
}
|
|
254
|
+
// Apps, dashboards, persona — see apps-api.ts. Returns true if handled.
|
|
255
|
+
if (path.startsWith('/api/apps') || path === '/api/persona') {
|
|
256
|
+
try {
|
|
257
|
+
const handled = await handleAppsApi({ req, res, url, path, projectRoot });
|
|
258
|
+
if (handled)
|
|
259
|
+
return;
|
|
260
|
+
}
|
|
261
|
+
catch (err) {
|
|
262
|
+
if (!res.headersSent) {
|
|
263
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
264
|
+
res.end(serializeJSON({ error: err.message }));
|
|
265
|
+
}
|
|
266
|
+
return;
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
// SSE endpoint for hot-reload file watching
|
|
270
|
+
if (req.method === 'GET' && path === '/api/watch') {
|
|
271
|
+
res.writeHead(200, {
|
|
272
|
+
'Content-Type': 'text/event-stream',
|
|
273
|
+
'Cache-Control': 'no-cache',
|
|
274
|
+
'Connection': 'keep-alive',
|
|
275
|
+
'X-Accel-Buffering': 'no',
|
|
276
|
+
});
|
|
277
|
+
res.write(': connected\n\n');
|
|
278
|
+
sseClients.add(res);
|
|
279
|
+
req.on('close', () => { sseClients.delete(res); });
|
|
280
|
+
return;
|
|
281
|
+
}
|
|
282
|
+
// ── dql-notebook file management API ─────────────────────────────────────
|
|
283
|
+
// GET /api/notebooks — list all .dql/.dqlnb files grouped by folder
|
|
284
|
+
// GET /api/notebook-content — read a file (?path=relative/path)
|
|
285
|
+
// POST /api/notebooks — create new notebook
|
|
286
|
+
// PUT /api/notebook-content — save file
|
|
287
|
+
// GET /api/schema — list data files for schema panel
|
|
288
|
+
if (req.method === 'GET' && path === '/api/notebooks') {
|
|
289
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
290
|
+
res.end(serializeJSON(scanNotebookFiles(projectRoot)));
|
|
291
|
+
return;
|
|
292
|
+
}
|
|
293
|
+
if (req.method === 'GET' && path === '/api/notebook-content') {
|
|
294
|
+
const filePath = url.searchParams.get('path');
|
|
295
|
+
if (!filePath) {
|
|
296
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
297
|
+
res.end(serializeJSON({ error: 'Missing path query parameter' }));
|
|
298
|
+
return;
|
|
299
|
+
}
|
|
300
|
+
const absPath = safeJoin(projectRoot, filePath);
|
|
301
|
+
if (!absPath || !existsSync(absPath)) {
|
|
302
|
+
res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
303
|
+
res.end(serializeJSON({ error: 'File not found' }));
|
|
304
|
+
return;
|
|
305
|
+
}
|
|
306
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
307
|
+
res.end(serializeJSON({ content: readFileSync(absPath, 'utf-8') }));
|
|
308
|
+
return;
|
|
309
|
+
}
|
|
310
|
+
if (req.method === 'POST' && path === '/api/notebooks') {
|
|
311
|
+
try {
|
|
312
|
+
const body = await readJSON(req);
|
|
313
|
+
const { name, template } = body;
|
|
314
|
+
if (!name || typeof name !== 'string') {
|
|
315
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
316
|
+
res.end(serializeJSON({ error: 'Missing notebook name' }));
|
|
317
|
+
return;
|
|
318
|
+
}
|
|
319
|
+
const slug = name.toLowerCase().replace(/[^a-z0-9]+/g, '_').replace(/^_+|_+$/g, '') || 'notebook';
|
|
320
|
+
const nbDir = join(projectRoot, 'notebooks');
|
|
321
|
+
mkdirSync(nbDir, { recursive: true });
|
|
322
|
+
const nbPath = join(nbDir, `${slug}.dqlnb`);
|
|
323
|
+
if (existsSync(nbPath)) {
|
|
324
|
+
res.writeHead(409, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
325
|
+
res.end(serializeJSON({ error: 'Notebook already exists' }));
|
|
326
|
+
return;
|
|
327
|
+
}
|
|
328
|
+
const content = buildNotebookTemplate(name, template ?? 'blank');
|
|
329
|
+
writeFileSync(nbPath, content, 'utf-8');
|
|
330
|
+
res.writeHead(201, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
331
|
+
res.end(serializeJSON({ path: `notebooks/${slug}.dqlnb`, content }));
|
|
332
|
+
}
|
|
333
|
+
catch (error) {
|
|
334
|
+
if (error instanceof DQLAccessDeniedError) {
|
|
335
|
+
res.writeHead(403, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
336
|
+
res.end(serializeJSON({ error: error.message, code: 'unauthorized' }));
|
|
337
|
+
return;
|
|
338
|
+
}
|
|
339
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
340
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
341
|
+
}
|
|
342
|
+
return;
|
|
343
|
+
}
|
|
344
|
+
if (req.method === 'PUT' && path === '/api/notebook-content') {
|
|
345
|
+
try {
|
|
346
|
+
const body = await readJSON(req);
|
|
347
|
+
const { path: filePath, content } = body;
|
|
348
|
+
if (!filePath || typeof content !== 'string') {
|
|
349
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
350
|
+
res.end(serializeJSON({ error: 'Missing path or content' }));
|
|
351
|
+
return;
|
|
352
|
+
}
|
|
353
|
+
const absPath = safeJoin(projectRoot, filePath);
|
|
354
|
+
if (!absPath) {
|
|
355
|
+
res.writeHead(403, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
356
|
+
res.end(serializeJSON({ error: 'Invalid path' }));
|
|
357
|
+
return;
|
|
358
|
+
}
|
|
359
|
+
mkdirSync(dirname(absPath), { recursive: true });
|
|
360
|
+
const toWrite = absPath.endsWith('.dql')
|
|
361
|
+
? canonicalizeSafe(content)
|
|
362
|
+
: absPath.endsWith('.dqlnb')
|
|
363
|
+
? canonicalizeNotebookSafe(content)
|
|
364
|
+
: content;
|
|
365
|
+
writeFileSync(absPath, toWrite, 'utf-8');
|
|
366
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
367
|
+
res.end(serializeJSON({ ok: true }));
|
|
368
|
+
}
|
|
369
|
+
catch (error) {
|
|
370
|
+
if (error instanceof DQLAccessDeniedError) {
|
|
371
|
+
res.writeHead(403, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
372
|
+
res.end(serializeJSON({ error: error.message, code: 'unauthorized' }));
|
|
373
|
+
return;
|
|
374
|
+
}
|
|
375
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
376
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
377
|
+
}
|
|
378
|
+
return;
|
|
379
|
+
}
|
|
380
|
+
// ── run snapshots (v0.11) ───────────────────────────────────────────────
|
|
381
|
+
// Captures executed notebook state (query results + timings) in a
|
|
382
|
+
// sibling `.run.json` so notebooks can show last-run output without
|
|
383
|
+
// re-executing after a reload. Snapshots are git-ignored by default.
|
|
384
|
+
if (req.method === 'GET' && path === '/api/run-snapshot') {
|
|
385
|
+
const notebookPath = url.searchParams.get('path') ?? '';
|
|
386
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
387
|
+
res.end(serializeJSON(readRunSnapshot(projectRoot, notebookPath)));
|
|
388
|
+
return;
|
|
389
|
+
}
|
|
390
|
+
if (req.method === 'PUT' && path === '/api/run-snapshot') {
|
|
391
|
+
try {
|
|
392
|
+
const body = await readJSON(req);
|
|
393
|
+
if (!body.path || typeof body.path !== 'string' || !body.snapshot) {
|
|
394
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
395
|
+
res.end(serializeJSON({ error: 'Missing path or snapshot' }));
|
|
396
|
+
return;
|
|
397
|
+
}
|
|
398
|
+
writeRunSnapshot(projectRoot, body.path, body.snapshot);
|
|
399
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
400
|
+
res.end(serializeJSON({ ok: true }));
|
|
401
|
+
}
|
|
402
|
+
catch (err) {
|
|
403
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
404
|
+
res.end(serializeJSON({ error: err instanceof Error ? err.message : String(err) }));
|
|
405
|
+
}
|
|
406
|
+
return;
|
|
407
|
+
}
|
|
408
|
+
// ── git read-only API (v0.11) ───────────────────────────────────────────
|
|
409
|
+
// GET /api/git/status — branch, clean, changed files
|
|
410
|
+
// GET /api/git/log — last N commits (?limit=20)
|
|
411
|
+
// GET /api/git/diff — unified diff for a single file (?path=relative/path)
|
|
412
|
+
if (req.method === 'GET' && path === '/api/git/status') {
|
|
413
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
414
|
+
res.end(serializeJSON(await readGitStatus(projectRoot)));
|
|
415
|
+
return;
|
|
416
|
+
}
|
|
417
|
+
if (req.method === 'GET' && path === '/api/git/log') {
|
|
418
|
+
const limit = Math.min(Number(url.searchParams.get('limit') ?? 20), 200);
|
|
419
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
420
|
+
res.end(serializeJSON(await readGitLog(projectRoot, limit)));
|
|
421
|
+
return;
|
|
422
|
+
}
|
|
423
|
+
if (req.method === 'GET' && path === '/api/git/diff') {
|
|
424
|
+
const filePath = url.searchParams.get('path') ?? '';
|
|
425
|
+
const staged = url.searchParams.get('staged') === 'true';
|
|
426
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
427
|
+
res.end(serializeJSON(await readGitDiff(projectRoot, filePath, staged)));
|
|
428
|
+
return;
|
|
429
|
+
}
|
|
430
|
+
if (req.method === 'GET' && path === '/api/git/branches') {
|
|
431
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
432
|
+
res.end(serializeJSON(await readGitBranches(projectRoot)));
|
|
433
|
+
return;
|
|
434
|
+
}
|
|
435
|
+
if (req.method === 'GET' && path === '/api/git/remote') {
|
|
436
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
437
|
+
res.end(serializeJSON(await readGitRemote(projectRoot)));
|
|
438
|
+
return;
|
|
439
|
+
}
|
|
440
|
+
if (req.method === 'POST' && path === '/api/git/stage') {
|
|
441
|
+
try {
|
|
442
|
+
const body = (await readJSON(req));
|
|
443
|
+
const result = await gitStage(projectRoot, body.paths ?? []);
|
|
444
|
+
res.writeHead(result.ok ? 200 : 400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
445
|
+
res.end(serializeJSON(result));
|
|
446
|
+
}
|
|
447
|
+
catch (e) {
|
|
448
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
449
|
+
res.end(serializeJSON({ ok: false, error: e instanceof Error ? e.message : String(e) }));
|
|
450
|
+
}
|
|
451
|
+
return;
|
|
452
|
+
}
|
|
453
|
+
if (req.method === 'POST' && path === '/api/git/unstage') {
|
|
454
|
+
try {
|
|
455
|
+
const body = (await readJSON(req));
|
|
456
|
+
const result = await gitUnstage(projectRoot, body.paths ?? []);
|
|
457
|
+
res.writeHead(result.ok ? 200 : 400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
458
|
+
res.end(serializeJSON(result));
|
|
459
|
+
}
|
|
460
|
+
catch (e) {
|
|
461
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
462
|
+
res.end(serializeJSON({ ok: false, error: e instanceof Error ? e.message : String(e) }));
|
|
463
|
+
}
|
|
464
|
+
return;
|
|
465
|
+
}
|
|
466
|
+
if (req.method === 'POST' && path === '/api/git/discard') {
|
|
467
|
+
try {
|
|
468
|
+
const body = (await readJSON(req));
|
|
469
|
+
const result = await gitDiscard(projectRoot, body.paths ?? []);
|
|
470
|
+
res.writeHead(result.ok ? 200 : 400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
471
|
+
res.end(serializeJSON(result));
|
|
472
|
+
}
|
|
473
|
+
catch (e) {
|
|
474
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
475
|
+
res.end(serializeJSON({ ok: false, error: e instanceof Error ? e.message : String(e) }));
|
|
476
|
+
}
|
|
477
|
+
return;
|
|
478
|
+
}
|
|
479
|
+
if (req.method === 'POST' && path === '/api/git/commit') {
|
|
480
|
+
try {
|
|
481
|
+
const body = (await readJSON(req));
|
|
482
|
+
const result = await gitCommit(projectRoot, body.message ?? '', body.stageAll === true);
|
|
483
|
+
res.writeHead(result.ok ? 200 : 400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
484
|
+
res.end(serializeJSON(result));
|
|
485
|
+
}
|
|
486
|
+
catch (e) {
|
|
487
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
488
|
+
res.end(serializeJSON({ ok: false, error: e instanceof Error ? e.message : String(e) }));
|
|
489
|
+
}
|
|
490
|
+
return;
|
|
491
|
+
}
|
|
492
|
+
if (req.method === 'POST' && path === '/api/git/push') {
|
|
493
|
+
try {
|
|
494
|
+
const result = await gitPush(projectRoot);
|
|
495
|
+
res.writeHead(result.ok ? 200 : 400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
496
|
+
res.end(serializeJSON(result));
|
|
497
|
+
}
|
|
498
|
+
catch (e) {
|
|
499
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
500
|
+
res.end(serializeJSON({ ok: false, error: e instanceof Error ? e.message : String(e) }));
|
|
501
|
+
}
|
|
502
|
+
return;
|
|
503
|
+
}
|
|
504
|
+
if (req.method === 'POST' && path === '/api/git/pull') {
|
|
505
|
+
try {
|
|
506
|
+
const result = await gitPull(projectRoot);
|
|
507
|
+
res.writeHead(result.ok ? 200 : 400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
508
|
+
res.end(serializeJSON(result));
|
|
509
|
+
}
|
|
510
|
+
catch (e) {
|
|
511
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
512
|
+
res.end(serializeJSON({ ok: false, error: e instanceof Error ? e.message : String(e) }));
|
|
513
|
+
}
|
|
514
|
+
return;
|
|
515
|
+
}
|
|
516
|
+
if (req.method === 'POST' && path === '/api/git/branch') {
|
|
517
|
+
try {
|
|
518
|
+
const body = (await readJSON(req));
|
|
519
|
+
const result = await gitCreateBranch(projectRoot, body.name ?? '', body.checkout !== false);
|
|
520
|
+
res.writeHead(result.ok ? 200 : 400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
521
|
+
res.end(serializeJSON(result));
|
|
522
|
+
}
|
|
523
|
+
catch (e) {
|
|
524
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
525
|
+
res.end(serializeJSON({ ok: false, error: e instanceof Error ? e.message : String(e) }));
|
|
526
|
+
}
|
|
527
|
+
return;
|
|
528
|
+
}
|
|
529
|
+
if (req.method === 'POST' && path === '/api/git/checkout') {
|
|
530
|
+
try {
|
|
531
|
+
const body = (await readJSON(req));
|
|
532
|
+
const result = await gitCheckout(projectRoot, body.name ?? '');
|
|
533
|
+
res.writeHead(result.ok ? 200 : 400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
534
|
+
res.end(serializeJSON(result));
|
|
535
|
+
}
|
|
536
|
+
catch (e) {
|
|
537
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
538
|
+
res.end(serializeJSON({ ok: false, error: e instanceof Error ? e.message : String(e) }));
|
|
539
|
+
}
|
|
540
|
+
return;
|
|
541
|
+
}
|
|
542
|
+
if (req.method === 'GET' && path === '/api/schema') {
|
|
543
|
+
try {
|
|
544
|
+
const dataFiles = scanDataFiles(projectRoot);
|
|
545
|
+
const { tables, columnsByPath } = await introspectSchema(executor, connection);
|
|
546
|
+
const dbTables = tables.map((t) => ({
|
|
547
|
+
name: t.path,
|
|
548
|
+
path: t.path,
|
|
549
|
+
columns: columnsByPath.get(t.path) ?? [],
|
|
550
|
+
source: 'database',
|
|
551
|
+
objectType: t.type,
|
|
552
|
+
}));
|
|
553
|
+
const seen = new Set(dataFiles.map((f) => f.name));
|
|
554
|
+
const merged = [
|
|
555
|
+
...dataFiles.map((f) => ({ ...f, source: 'file' })),
|
|
556
|
+
...dbTables.filter((t) => !seen.has(t.name)),
|
|
557
|
+
];
|
|
558
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
559
|
+
res.end(serializeJSON(merged));
|
|
560
|
+
}
|
|
561
|
+
catch (error) {
|
|
562
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
563
|
+
console.warn(`[dql] /api/schema introspection failed: ${message}`);
|
|
564
|
+
const fallback = scanDataFiles(projectRoot).map((f) => ({ ...f, source: 'file' }));
|
|
565
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
566
|
+
res.end(serializeJSON({ error: message, fallback }));
|
|
567
|
+
}
|
|
568
|
+
return;
|
|
569
|
+
}
|
|
570
|
+
if (req.method === 'POST' && path === '/api/blocks') {
|
|
571
|
+
try {
|
|
572
|
+
const body = await readJSON(req);
|
|
573
|
+
const { name, domain, content, description, tags, metricRefs, template, } = body;
|
|
574
|
+
if (!name || typeof name !== 'string') {
|
|
575
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
576
|
+
res.end(serializeJSON({ error: 'Missing block name' }));
|
|
577
|
+
return;
|
|
578
|
+
}
|
|
579
|
+
const created = createBlockArtifacts(projectRoot, {
|
|
580
|
+
name,
|
|
581
|
+
domain,
|
|
582
|
+
content,
|
|
583
|
+
description,
|
|
584
|
+
tags,
|
|
585
|
+
metricRefs,
|
|
586
|
+
template,
|
|
587
|
+
});
|
|
588
|
+
res.writeHead(201, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
589
|
+
res.end(serializeJSON(created));
|
|
590
|
+
}
|
|
591
|
+
catch (error) {
|
|
592
|
+
if (error instanceof Error && error.message === 'BLOCK_EXISTS') {
|
|
593
|
+
res.writeHead(409, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
594
|
+
res.end(serializeJSON({ error: 'Block already exists' }));
|
|
595
|
+
return;
|
|
596
|
+
}
|
|
597
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
598
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
599
|
+
}
|
|
600
|
+
return;
|
|
601
|
+
}
|
|
602
|
+
if (req.method === 'POST' && path === '/api/blocks/save-from-cell') {
|
|
603
|
+
try {
|
|
604
|
+
const body = await readJSON(req);
|
|
605
|
+
const { name, domain, owner, content, description, tags, metricRefs, template, llmContext, examples, invariants, } = body;
|
|
606
|
+
if (!name || typeof name !== 'string' || !content || typeof content !== 'string') {
|
|
607
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
608
|
+
res.end(serializeJSON({ error: 'name and content are required' }));
|
|
609
|
+
return;
|
|
610
|
+
}
|
|
611
|
+
const missing = [];
|
|
612
|
+
if (!owner || !owner.trim())
|
|
613
|
+
missing.push('owner');
|
|
614
|
+
if (!domain || !domain.trim())
|
|
615
|
+
missing.push('domain');
|
|
616
|
+
if (!description || !description.trim())
|
|
617
|
+
missing.push('description');
|
|
618
|
+
if (missing.length > 0) {
|
|
619
|
+
res.writeHead(422, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
620
|
+
res.end(serializeJSON({
|
|
621
|
+
error: `Block is missing required governance fields: ${missing.join(', ')}`,
|
|
622
|
+
missing,
|
|
623
|
+
}));
|
|
624
|
+
return;
|
|
625
|
+
}
|
|
626
|
+
const created = createBlockArtifacts(projectRoot, {
|
|
627
|
+
name,
|
|
628
|
+
domain,
|
|
629
|
+
owner,
|
|
630
|
+
content,
|
|
631
|
+
description,
|
|
632
|
+
tags,
|
|
633
|
+
metricRefs,
|
|
634
|
+
template,
|
|
635
|
+
llmContext,
|
|
636
|
+
examples,
|
|
637
|
+
invariants,
|
|
638
|
+
gitMetadata: readGitMetadata(projectRoot),
|
|
639
|
+
});
|
|
640
|
+
res.writeHead(201, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
641
|
+
res.end(serializeJSON(created));
|
|
642
|
+
}
|
|
643
|
+
catch (error) {
|
|
644
|
+
if (error instanceof Error && error.message === 'BLOCK_EXISTS') {
|
|
645
|
+
res.writeHead(409, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
646
|
+
res.end(serializeJSON({ error: 'Block already exists' }));
|
|
647
|
+
return;
|
|
648
|
+
}
|
|
649
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
650
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
651
|
+
}
|
|
652
|
+
return;
|
|
653
|
+
}
|
|
654
|
+
if (req.method === 'GET' && path === '/api/blocks/templates') {
|
|
655
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
656
|
+
res.end(serializeJSON({ templates: listBlockTemplates() }));
|
|
657
|
+
return;
|
|
658
|
+
}
|
|
659
|
+
// ── Block library (list all blocks with metadata) ────────────────────
|
|
660
|
+
if (req.method === 'GET' && path === '/api/blocks/library') {
|
|
661
|
+
try {
|
|
662
|
+
const blocksDir = join(projectRoot, 'blocks');
|
|
663
|
+
const blocks = [];
|
|
664
|
+
if (existsSync(blocksDir)) {
|
|
665
|
+
const scanDir = (dir) => {
|
|
666
|
+
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
667
|
+
if (entry.isDirectory()) {
|
|
668
|
+
scanDir(join(dir, entry.name));
|
|
669
|
+
}
|
|
670
|
+
else if (entry.name.endsWith('.dql')) {
|
|
671
|
+
const filePath = join(dir, entry.name);
|
|
672
|
+
const relPath = relative(projectRoot, filePath);
|
|
673
|
+
try {
|
|
674
|
+
const source = readFileSync(filePath, 'utf-8');
|
|
675
|
+
const stat = statSync(filePath);
|
|
676
|
+
// Quick regex parse for key block fields
|
|
677
|
+
const nameMatch = /block\s+"([^"]+)"/.exec(source);
|
|
678
|
+
const domainMatch = /domain\s*=\s*"([^"]+)"/.exec(source);
|
|
679
|
+
const statusMatch = /status\s*=\s*"([^"]+)"/.exec(source);
|
|
680
|
+
const ownerMatch = /owner\s*=\s*"([^"]+)"/.exec(source);
|
|
681
|
+
const descMatch = /description\s*=\s*"([^"]+)"/.exec(source);
|
|
682
|
+
const tagsMatch = /tags\s*=\s*\[([^\]]*)\]/.exec(source);
|
|
683
|
+
const parsedTags = tagsMatch
|
|
684
|
+
? tagsMatch[1].split(',').map((tag) => tag.trim().replace(/^"|"$/g, '')).filter(Boolean)
|
|
685
|
+
: [];
|
|
686
|
+
const llmMatch = /llmContext\s*=\s*"((?:[^"\\]|\\.)*)"/.exec(source);
|
|
687
|
+
blocks.push({
|
|
688
|
+
name: nameMatch?.[1] ?? entry.name.replace('.dql', ''),
|
|
689
|
+
domain: domainMatch?.[1] ?? 'uncategorized',
|
|
690
|
+
status: statusMatch?.[1] ?? 'draft',
|
|
691
|
+
owner: ownerMatch?.[1] ?? null,
|
|
692
|
+
tags: parsedTags,
|
|
693
|
+
path: relPath,
|
|
694
|
+
lastModified: stat.mtime.toISOString(),
|
|
695
|
+
description: descMatch?.[1] ?? '',
|
|
696
|
+
llmContext: llmMatch?.[1] ?? null,
|
|
697
|
+
});
|
|
698
|
+
}
|
|
699
|
+
catch { /* skip unreadable files */ }
|
|
700
|
+
}
|
|
701
|
+
}
|
|
702
|
+
};
|
|
703
|
+
scanDir(blocksDir);
|
|
704
|
+
}
|
|
705
|
+
blocks.sort((a, b) => new Date(b.lastModified).getTime() - new Date(a.lastModified).getTime());
|
|
706
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
707
|
+
res.end(serializeJSON({ blocks }));
|
|
708
|
+
}
|
|
709
|
+
catch (error) {
|
|
710
|
+
if (error instanceof DQLAccessDeniedError) {
|
|
711
|
+
res.writeHead(403, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
712
|
+
res.end(serializeJSON({ error: error.message, code: 'unauthorized' }));
|
|
713
|
+
return;
|
|
714
|
+
}
|
|
715
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
716
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
717
|
+
}
|
|
718
|
+
return;
|
|
719
|
+
}
|
|
720
|
+
// ── Apps (App artifact listing for notebook AppsPanel) ────────────────
|
|
721
|
+
if (req.method === 'GET' && path === '/api/apps') {
|
|
722
|
+
try {
|
|
723
|
+
const appsRoot = join(projectRoot, 'apps');
|
|
724
|
+
const apps = [];
|
|
725
|
+
const listFilesByExt = (dir, ext) => {
|
|
726
|
+
const out = [];
|
|
727
|
+
try {
|
|
728
|
+
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
729
|
+
const full = join(dir, entry.name);
|
|
730
|
+
if (entry.isDirectory()) {
|
|
731
|
+
out.push(...listFilesByExt(full, ext).map((n) => `${entry.name}/${n}`));
|
|
732
|
+
}
|
|
733
|
+
else if (entry.isFile() && entry.name.endsWith(ext)) {
|
|
734
|
+
out.push(entry.name);
|
|
735
|
+
}
|
|
736
|
+
}
|
|
737
|
+
}
|
|
738
|
+
catch { /* dir missing; return [] */ }
|
|
739
|
+
return out;
|
|
740
|
+
};
|
|
741
|
+
if (existsSync(appsRoot)) {
|
|
742
|
+
for (const entry of readdirSync(appsRoot, { withFileTypes: true })) {
|
|
743
|
+
if (!entry.isDirectory() || !entry.name.endsWith('.dql-app'))
|
|
744
|
+
continue;
|
|
745
|
+
const appDir = join(appsRoot, entry.name);
|
|
746
|
+
try {
|
|
747
|
+
const raw = readFileSync(join(appDir, 'app.yml'), 'utf-8');
|
|
748
|
+
const manifest = loadYaml(raw);
|
|
749
|
+
if (!manifest || !manifest.name || !manifest.domain)
|
|
750
|
+
continue;
|
|
751
|
+
apps.push({
|
|
752
|
+
path: relative(projectRoot, appDir),
|
|
753
|
+
manifest,
|
|
754
|
+
notebooks: listFilesByExt(join(appDir, 'notebooks'), '.dqlnb'),
|
|
755
|
+
dashboards: listFilesByExt(join(appDir, 'dashboards'), '.dql'),
|
|
756
|
+
hasDigest: existsSync(join(appDir, 'digest.dql')),
|
|
757
|
+
});
|
|
758
|
+
}
|
|
759
|
+
catch { /* skip unreadable apps */ }
|
|
760
|
+
}
|
|
761
|
+
}
|
|
762
|
+
apps.sort((a, b) => a.manifest.name.localeCompare(b.manifest.name));
|
|
763
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
764
|
+
res.end(serializeJSON({ apps }));
|
|
765
|
+
}
|
|
766
|
+
catch (error) {
|
|
767
|
+
const status = error instanceof DQLAccessDeniedError ? 403 : 500;
|
|
768
|
+
res.writeHead(status, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
769
|
+
res.end(serializeJSON({
|
|
770
|
+
error: error instanceof Error ? error.message : String(error),
|
|
771
|
+
...(status === 403 ? { code: 'unauthorized' } : {}),
|
|
772
|
+
}));
|
|
773
|
+
}
|
|
774
|
+
return;
|
|
775
|
+
}
|
|
776
|
+
// ── Block status update ──────────────────────────────────────────────
|
|
777
|
+
if (req.method === 'POST' && path === '/api/blocks/status') {
|
|
778
|
+
try {
|
|
779
|
+
const body = await readJSON(req);
|
|
780
|
+
const blockPath = body.path;
|
|
781
|
+
const newStatus = body.newStatus;
|
|
782
|
+
const validStatuses = ['draft', 'review', 'certified', 'deprecated'];
|
|
783
|
+
if (!validStatuses.includes(newStatus)) {
|
|
784
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
785
|
+
res.end(serializeJSON({ error: `Status must be one of: ${validStatuses.join(', ')}` }));
|
|
786
|
+
return;
|
|
787
|
+
}
|
|
788
|
+
const absPath = resolve(projectRoot, blockPath);
|
|
789
|
+
if (!existsSync(absPath)) {
|
|
790
|
+
res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
791
|
+
res.end(serializeJSON({ error: 'Block file not found' }));
|
|
792
|
+
return;
|
|
793
|
+
}
|
|
794
|
+
let source = readFileSync(absPath, 'utf-8');
|
|
795
|
+
// Update or insert status field
|
|
796
|
+
if (/status\s*=\s*"[^"]*"/.test(source)) {
|
|
797
|
+
source = source.replace(/status\s*=\s*"[^"]*"/, `status = "${newStatus}"`);
|
|
798
|
+
}
|
|
799
|
+
else {
|
|
800
|
+
// Insert after first { in block declaration
|
|
801
|
+
source = source.replace(/block\s+"[^"]*"\s*\{/, (match) => `${match}\n status = "${newStatus}"`);
|
|
802
|
+
}
|
|
803
|
+
writeFileSync(absPath, source, 'utf-8');
|
|
804
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
805
|
+
res.end(serializeJSON({ ok: true, status: newStatus }));
|
|
806
|
+
}
|
|
807
|
+
catch (error) {
|
|
808
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
809
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
810
|
+
}
|
|
811
|
+
return;
|
|
812
|
+
}
|
|
813
|
+
// ── Block version history (git log) ──────────────────────────────────
|
|
814
|
+
if (req.method === 'GET' && path === '/api/blocks/history') {
|
|
815
|
+
try {
|
|
816
|
+
const blockPath = url.searchParams.get('path');
|
|
817
|
+
if (!blockPath) {
|
|
818
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
819
|
+
res.end(serializeJSON({ error: 'path parameter is required' }));
|
|
820
|
+
return;
|
|
821
|
+
}
|
|
822
|
+
const { execSync } = await import('node:child_process');
|
|
823
|
+
const gitLog = execSync(`git log --format="%H|||%ai|||%an|||%s" -20 -- "${blockPath}"`, { cwd: projectRoot, encoding: 'utf-8', timeout: 10000 }).trim();
|
|
824
|
+
const entries = gitLog
|
|
825
|
+
? gitLog.split('\n').map((line) => {
|
|
826
|
+
const [hash, date, author, message] = line.split('|||');
|
|
827
|
+
return { hash, date, author, message };
|
|
828
|
+
})
|
|
829
|
+
: [];
|
|
830
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
831
|
+
res.end(serializeJSON({ entries }));
|
|
832
|
+
}
|
|
833
|
+
catch (error) {
|
|
834
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
835
|
+
res.end(serializeJSON({ entries: [] }));
|
|
836
|
+
}
|
|
837
|
+
return;
|
|
838
|
+
}
|
|
839
|
+
// ── Block body (re-read from disk, used by bound-cell refresh) ──────
|
|
840
|
+
if (req.method === 'GET' && path === '/api/blocks/body') {
|
|
841
|
+
try {
|
|
842
|
+
const blockPath = url.searchParams.get('path');
|
|
843
|
+
if (!blockPath) {
|
|
844
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
845
|
+
res.end(serializeJSON({ error: 'path parameter is required' }));
|
|
846
|
+
return;
|
|
847
|
+
}
|
|
848
|
+
const absolutePath = resolve(projectRoot, blockPath);
|
|
849
|
+
if (!absolutePath.startsWith(projectRoot + '/') && absolutePath !== projectRoot) {
|
|
850
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
851
|
+
res.end(serializeJSON({ error: 'path escapes project root' }));
|
|
852
|
+
return;
|
|
853
|
+
}
|
|
854
|
+
if (!existsSync(absolutePath)) {
|
|
855
|
+
res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
856
|
+
res.end(serializeJSON({ error: 'block not found' }));
|
|
857
|
+
return;
|
|
858
|
+
}
|
|
859
|
+
const body = readFileSync(absolutePath, 'utf-8');
|
|
860
|
+
let commitSha = null;
|
|
861
|
+
try {
|
|
862
|
+
const { execSync } = await import('node:child_process');
|
|
863
|
+
const sha = execSync(`git log -1 --format=%H -- "${blockPath}"`, {
|
|
864
|
+
cwd: projectRoot,
|
|
865
|
+
encoding: 'utf-8',
|
|
866
|
+
stdio: ['ignore', 'pipe', 'ignore'],
|
|
867
|
+
timeout: 5000,
|
|
868
|
+
}).trim();
|
|
869
|
+
commitSha = sha.length > 0 ? sha : null;
|
|
870
|
+
}
|
|
871
|
+
catch {
|
|
872
|
+
commitSha = null;
|
|
873
|
+
}
|
|
874
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
875
|
+
res.end(serializeJSON({ path: blockPath, body, commitSha }));
|
|
876
|
+
}
|
|
877
|
+
catch (error) {
|
|
878
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
879
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
880
|
+
res.end(serializeJSON({ error: message }));
|
|
881
|
+
}
|
|
882
|
+
return;
|
|
883
|
+
}
|
|
884
|
+
// ── Run block tests ────────────────────────────────────────────────
|
|
885
|
+
if (req.method === 'POST' && path === '/api/blocks/run-tests') {
|
|
886
|
+
try {
|
|
887
|
+
const body = await readJSON(req);
|
|
888
|
+
const source = body.source;
|
|
889
|
+
if (!source) {
|
|
890
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
891
|
+
res.end(serializeJSON({ error: 'source is required' }));
|
|
892
|
+
return;
|
|
893
|
+
}
|
|
894
|
+
// Parse the block to extract tests and query SQL
|
|
895
|
+
const parser = new Parser(source, '<run-tests>');
|
|
896
|
+
const ast = parser.parse();
|
|
897
|
+
const blockNode = ast.statements.find((n) => n.kind === 'BlockDecl');
|
|
898
|
+
if (!blockNode) {
|
|
899
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
900
|
+
res.end(serializeJSON({ error: 'No block declaration found in source' }));
|
|
901
|
+
return;
|
|
902
|
+
}
|
|
903
|
+
const testNodes = blockNode.tests ?? [];
|
|
904
|
+
if (testNodes.length === 0) {
|
|
905
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
906
|
+
res.end(serializeJSON({ assertions: [], passed: 0, failed: 0, duration: 0 }));
|
|
907
|
+
return;
|
|
908
|
+
}
|
|
909
|
+
// Get the block's base SQL
|
|
910
|
+
const baseSql = blockNode.query?.rawSQL?.trim() ?? '';
|
|
911
|
+
if (!baseSql) {
|
|
912
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
913
|
+
res.end(serializeJSON({ error: 'Block has no query SQL to test against' }));
|
|
914
|
+
return;
|
|
915
|
+
}
|
|
916
|
+
const resolvedSql = resolveProjectRelativeSqlPaths(baseSql, projectRoot, projectConfig.dataDir);
|
|
917
|
+
// Build and run assertions
|
|
918
|
+
const start = Date.now();
|
|
919
|
+
const results = [];
|
|
920
|
+
for (const test of testNodes) {
|
|
921
|
+
const field = test.field;
|
|
922
|
+
const op = test.operator;
|
|
923
|
+
const expected = test.expected;
|
|
924
|
+
// Extract the expected value from the AST node
|
|
925
|
+
const expectedValue = typeof expected === 'object' && expected !== null
|
|
926
|
+
? (expected.value ?? String(expected))
|
|
927
|
+
: expected;
|
|
928
|
+
// Build a SQL query that computes the aggregate for this assertion
|
|
929
|
+
const testSql = `SELECT ${field} AS test_value FROM (${resolvedSql}) AS __test_block`;
|
|
930
|
+
try {
|
|
931
|
+
const result = await executor.executeQuery(testSql, [], {}, connection);
|
|
932
|
+
const actualRaw = result.rows?.[0];
|
|
933
|
+
const actual = actualRaw ? Object.values(actualRaw)[0] : undefined;
|
|
934
|
+
const actualNum = Number(actual);
|
|
935
|
+
const expectedNum = Number(expectedValue);
|
|
936
|
+
let passed = false;
|
|
937
|
+
switch (op) {
|
|
938
|
+
case '>':
|
|
939
|
+
passed = actualNum > expectedNum;
|
|
940
|
+
break;
|
|
941
|
+
case '<':
|
|
942
|
+
passed = actualNum < expectedNum;
|
|
943
|
+
break;
|
|
944
|
+
case '>=':
|
|
945
|
+
passed = actualNum >= expectedNum;
|
|
946
|
+
break;
|
|
947
|
+
case '<=':
|
|
948
|
+
passed = actualNum <= expectedNum;
|
|
949
|
+
break;
|
|
950
|
+
case '==':
|
|
951
|
+
passed = String(actual) === String(expectedValue);
|
|
952
|
+
break;
|
|
953
|
+
case '!=':
|
|
954
|
+
passed = String(actual) !== String(expectedValue);
|
|
955
|
+
break;
|
|
956
|
+
default: passed = false;
|
|
957
|
+
}
|
|
958
|
+
results.push({ field, operator: op, expected: String(expectedValue), passed, actual: String(actual ?? '') });
|
|
959
|
+
}
|
|
960
|
+
catch (err) {
|
|
961
|
+
results.push({ field, operator: op, expected: String(expectedValue), passed: false, actual: `Error: ${err instanceof Error ? err.message : String(err)}` });
|
|
962
|
+
}
|
|
963
|
+
}
|
|
964
|
+
const duration = Date.now() - start;
|
|
965
|
+
const passed = results.filter((r) => r.passed).length;
|
|
966
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
967
|
+
res.end(serializeJSON({ assertions: results, passed, failed: results.length - passed, duration }));
|
|
968
|
+
}
|
|
969
|
+
catch (error) {
|
|
970
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
971
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
972
|
+
}
|
|
973
|
+
return;
|
|
974
|
+
}
|
|
975
|
+
if (req.method === 'GET' && path === '/api/block-studio/catalog') {
|
|
976
|
+
try {
|
|
977
|
+
const cfg = loadProjectConfig(projectRoot);
|
|
978
|
+
const connections = cfg.connections ?? {};
|
|
979
|
+
if (Object.keys(connections).length === 0 && cfg.defaultConnection) {
|
|
980
|
+
connections.default = cfg.defaultConnection;
|
|
981
|
+
}
|
|
982
|
+
const defaultKey = cfg.defaultConnection ? 'default' : Object.keys(connections)[0] ?? 'default';
|
|
983
|
+
const userPrefs = readUserPrefs(userPrefsPath);
|
|
984
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
985
|
+
res.end(serializeJSON({
|
|
986
|
+
semanticTree: semanticLayer ? buildSemanticTree(semanticLayer, semanticImportManifest) : null,
|
|
987
|
+
databaseTree: await buildDatabaseSchemaTree(projectRoot, executor, connection),
|
|
988
|
+
connection: {
|
|
989
|
+
default: defaultKey,
|
|
990
|
+
current: defaultKey,
|
|
991
|
+
connections,
|
|
992
|
+
},
|
|
993
|
+
favorites: userPrefs.favorites,
|
|
994
|
+
recentlyUsed: userPrefs.recentlyUsed,
|
|
995
|
+
}));
|
|
996
|
+
}
|
|
997
|
+
catch (error) {
|
|
998
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
999
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
1000
|
+
}
|
|
1001
|
+
return;
|
|
1002
|
+
}
|
|
1003
|
+
if (req.method === 'GET' && path === '/api/block-studio/open') {
|
|
1004
|
+
try {
|
|
1005
|
+
const relativePath = url.searchParams.get('path');
|
|
1006
|
+
if (!relativePath) {
|
|
1007
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1008
|
+
res.end(serializeJSON({ error: 'Missing block path.' }));
|
|
1009
|
+
return;
|
|
1010
|
+
}
|
|
1011
|
+
const payload = openBlockStudioDocument(projectRoot, relativePath, semanticLayer);
|
|
1012
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1013
|
+
res.end(serializeJSON(payload));
|
|
1014
|
+
}
|
|
1015
|
+
catch (error) {
|
|
1016
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1017
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
1018
|
+
}
|
|
1019
|
+
return;
|
|
1020
|
+
}
|
|
1021
|
+
if (req.method === 'POST' && path === '/api/block-studio/validate') {
|
|
1022
|
+
try {
|
|
1023
|
+
const body = await readJSON(req);
|
|
1024
|
+
const source = typeof body.source === 'string' ? body.source : '';
|
|
1025
|
+
const validation = validateBlockStudioSource(source, semanticLayer);
|
|
1026
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1027
|
+
res.end(serializeJSON(validation));
|
|
1028
|
+
}
|
|
1029
|
+
catch (error) {
|
|
1030
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1031
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
1032
|
+
}
|
|
1033
|
+
return;
|
|
1034
|
+
}
|
|
1035
|
+
if (req.method === 'POST' && path === '/api/block-studio/run') {
|
|
1036
|
+
try {
|
|
1037
|
+
const body = await readJSON(req);
|
|
1038
|
+
const source = typeof body.source === 'string' ? body.source : '';
|
|
1039
|
+
const targetConnection = isConnectionConfig(body.connection) ? body.connection : connection;
|
|
1040
|
+
let tableMapping;
|
|
1041
|
+
if (semanticLayer) {
|
|
1042
|
+
try {
|
|
1043
|
+
const tablesResult = await executor.executeQuery(`SELECT table_schema, table_name
|
|
1044
|
+
FROM information_schema.tables
|
|
1045
|
+
WHERE table_schema NOT IN ('information_schema', 'pg_catalog')`, [], {}, targetConnection);
|
|
1046
|
+
tableMapping = buildSemanticTableMapping(semanticLayer, tablesResult.rows);
|
|
1047
|
+
}
|
|
1048
|
+
catch {
|
|
1049
|
+
tableMapping = undefined;
|
|
1050
|
+
}
|
|
1051
|
+
}
|
|
1052
|
+
const semanticCompose = semanticLayer
|
|
1053
|
+
? composeSemanticBlockSql(source, semanticLayer, {
|
|
1054
|
+
driver: targetConnection.driver,
|
|
1055
|
+
tableMapping,
|
|
1056
|
+
projectRoot,
|
|
1057
|
+
projectConfig,
|
|
1058
|
+
detectedProvider: semanticDetectedProvider,
|
|
1059
|
+
})
|
|
1060
|
+
: null;
|
|
1061
|
+
const validation = validateBlockStudioSource(source, semanticLayer);
|
|
1062
|
+
const executableSql = semanticCompose?.sql ?? validation.executableSql;
|
|
1063
|
+
if (!executableSql) {
|
|
1064
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1065
|
+
const message = semanticCompose?.diagnostics.find((item) => item.severity === 'error')?.message
|
|
1066
|
+
?? validation.diagnostics.find((item) => item.severity === 'error')?.message
|
|
1067
|
+
?? 'No executable SQL found in block source.';
|
|
1068
|
+
res.end(serializeJSON({ error: message, diagnostics: validation.diagnostics }));
|
|
1069
|
+
return;
|
|
1070
|
+
}
|
|
1071
|
+
const plan = buildExecutionPlan({ id: 'block-studio', type: 'dql', source, title: 'Block Studio' }, { semanticLayer, driver: targetConnection.driver });
|
|
1072
|
+
const sql = resolveProjectRelativeSqlPaths(semanticCompose?.sql ?? plan?.sql ?? executableSql, projectRoot, projectConfig.dataDir);
|
|
1073
|
+
const result = await executor.executeQuery(sql, plan?.sqlParams ?? [], runtimeVariables(plan?.variables ?? {}), targetConnection);
|
|
1074
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1075
|
+
res.end(serializeJSON({
|
|
1076
|
+
sql: plan?.sql ?? executableSql,
|
|
1077
|
+
result: normalizeQueryResult(result),
|
|
1078
|
+
chartConfig: plan?.chartConfig ?? validation.chartConfig ?? null,
|
|
1079
|
+
}));
|
|
1080
|
+
}
|
|
1081
|
+
catch (error) {
|
|
1082
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1083
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
1084
|
+
}
|
|
1085
|
+
return;
|
|
1086
|
+
}
|
|
1087
|
+
if (req.method === 'POST' && path === '/api/block-studio/save') {
|
|
1088
|
+
try {
|
|
1089
|
+
const body = await readJSON(req);
|
|
1090
|
+
const source = typeof body.source === 'string' ? body.source : '';
|
|
1091
|
+
const metadata = body.metadata && typeof body.metadata === 'object'
|
|
1092
|
+
? body.metadata
|
|
1093
|
+
: {};
|
|
1094
|
+
if (!source.trim()) {
|
|
1095
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1096
|
+
res.end(serializeJSON({ error: 'Block source is required.' }));
|
|
1097
|
+
return;
|
|
1098
|
+
}
|
|
1099
|
+
if (!metadata.name || typeof metadata.name !== 'string') {
|
|
1100
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1101
|
+
res.end(serializeJSON({ error: 'Block name is required.' }));
|
|
1102
|
+
return;
|
|
1103
|
+
}
|
|
1104
|
+
const savedPath = saveBlockStudioArtifacts(projectRoot, {
|
|
1105
|
+
currentPath: typeof body.path === 'string' ? body.path : undefined,
|
|
1106
|
+
source,
|
|
1107
|
+
name: metadata.name,
|
|
1108
|
+
domain: metadata.domain,
|
|
1109
|
+
description: metadata.description,
|
|
1110
|
+
owner: metadata.owner,
|
|
1111
|
+
tags: Array.isArray(metadata.tags) ? metadata.tags.map(String) : [],
|
|
1112
|
+
});
|
|
1113
|
+
const payload = openBlockStudioDocument(projectRoot, savedPath, semanticLayer);
|
|
1114
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1115
|
+
res.end(serializeJSON(payload));
|
|
1116
|
+
}
|
|
1117
|
+
catch (error) {
|
|
1118
|
+
if (error instanceof Error && error.message === 'BLOCK_EXISTS') {
|
|
1119
|
+
res.writeHead(409, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1120
|
+
res.end(serializeJSON({ error: 'Block already exists' }));
|
|
1121
|
+
return;
|
|
1122
|
+
}
|
|
1123
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1124
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
1125
|
+
}
|
|
1126
|
+
return;
|
|
1127
|
+
}
|
|
1128
|
+
if (req.method === 'GET' && path === '/api/connections') {
|
|
1129
|
+
const cfg = loadProjectConfig(projectRoot);
|
|
1130
|
+
const raw = cfg;
|
|
1131
|
+
const connections = raw.connections ?? {};
|
|
1132
|
+
// If no explicit connections map, surface the defaultConnection as "default"
|
|
1133
|
+
if (Object.keys(connections).length === 0 && cfg.defaultConnection) {
|
|
1134
|
+
connections['default'] = cfg.defaultConnection;
|
|
1135
|
+
}
|
|
1136
|
+
const defaultKey = raw.defaultConnection
|
|
1137
|
+
? 'default'
|
|
1138
|
+
: Object.keys(connections)[0] ?? 'default';
|
|
1139
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1140
|
+
res.end(serializeJSON({ default: defaultKey, connections }));
|
|
1141
|
+
return;
|
|
1142
|
+
}
|
|
1143
|
+
// Save/update connections
|
|
1144
|
+
if (req.method === 'PUT' && path === '/api/connections') {
|
|
1145
|
+
try {
|
|
1146
|
+
const body = await readJSON(req);
|
|
1147
|
+
const configPath = join(projectRoot, 'dql.config.json');
|
|
1148
|
+
let raw = {};
|
|
1149
|
+
if (existsSync(configPath)) {
|
|
1150
|
+
raw = JSON.parse(readFileSync(configPath, 'utf-8'));
|
|
1151
|
+
}
|
|
1152
|
+
if (body.connections && typeof body.connections === 'object') {
|
|
1153
|
+
raw.connections = body.connections;
|
|
1154
|
+
}
|
|
1155
|
+
writeFileSync(configPath, JSON.stringify(raw, null, 2) + '\n', 'utf-8');
|
|
1156
|
+
// Hot-swap: re-read the config and re-initialize the active connection
|
|
1157
|
+
projectConfig = loadProjectConfig(projectRoot);
|
|
1158
|
+
const newDefault = projectConfig.defaultConnection;
|
|
1159
|
+
if (newDefault) {
|
|
1160
|
+
connection = normalizeProjectConnection(newDefault, projectRoot);
|
|
1161
|
+
// Auto-register data files if DuckDB/file driver
|
|
1162
|
+
if (connection.driver === 'file' || connection.driver === 'duckdb') {
|
|
1163
|
+
const dataDir = projectConfig.dataDir
|
|
1164
|
+
? resolve(projectRoot, projectConfig.dataDir)
|
|
1165
|
+
: join(projectRoot, 'data');
|
|
1166
|
+
if (existsSync(dataDir)) {
|
|
1167
|
+
try {
|
|
1168
|
+
const files = readdirSync(dataDir, { withFileTypes: true })
|
|
1169
|
+
.filter((e) => e.isFile() && /\.(csv|parquet)$/i.test(e.name));
|
|
1170
|
+
for (const file of files) {
|
|
1171
|
+
const tableName = file.name.replace(/\.(csv|parquet)$/i, '');
|
|
1172
|
+
const absPath = join(dataDir, file.name).replaceAll('\\', '/');
|
|
1173
|
+
const reader = file.name.endsWith('.parquet') ? 'read_parquet' : 'read_csv_auto';
|
|
1174
|
+
const ddl = `CREATE OR REPLACE VIEW "${tableName}" AS SELECT * FROM ${reader}('${absPath}')`;
|
|
1175
|
+
try {
|
|
1176
|
+
await executor.executeQuery(ddl, [], {}, connection);
|
|
1177
|
+
}
|
|
1178
|
+
catch { /* non-fatal */ }
|
|
1179
|
+
}
|
|
1180
|
+
}
|
|
1181
|
+
catch { /* non-fatal */ }
|
|
1182
|
+
}
|
|
1183
|
+
}
|
|
1184
|
+
}
|
|
1185
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1186
|
+
res.end(serializeJSON({ ok: true }));
|
|
1187
|
+
}
|
|
1188
|
+
catch (error) {
|
|
1189
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1190
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
1191
|
+
}
|
|
1192
|
+
return;
|
|
1193
|
+
}
|
|
1194
|
+
// ── Semantic layer discovery API ─────────────────────────────────────────
|
|
1195
|
+
if (req.method === 'GET' && path === '/api/semantic-layer') {
|
|
1196
|
+
const userPrefs = readUserPrefs(userPrefsPath);
|
|
1197
|
+
if (!semanticLayer) {
|
|
1198
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1199
|
+
res.end(serializeJSON({
|
|
1200
|
+
available: false,
|
|
1201
|
+
provider: projectConfig.semanticLayer?.provider ?? semanticDetectedProvider ?? null,
|
|
1202
|
+
errors: semanticLayerErrors,
|
|
1203
|
+
metrics: [],
|
|
1204
|
+
measures: [],
|
|
1205
|
+
dimensions: [],
|
|
1206
|
+
timeDimensions: [],
|
|
1207
|
+
entities: [],
|
|
1208
|
+
hierarchies: [],
|
|
1209
|
+
semanticModels: [],
|
|
1210
|
+
savedQueries: [],
|
|
1211
|
+
domains: [],
|
|
1212
|
+
tags: [],
|
|
1213
|
+
favorites: userPrefs.favorites,
|
|
1214
|
+
recentlyUsed: userPrefs.recentlyUsed,
|
|
1215
|
+
lastSyncTime: semanticLastSyncTime,
|
|
1216
|
+
}));
|
|
1217
|
+
return;
|
|
1218
|
+
}
|
|
1219
|
+
const metrics = semanticLayer.listMetrics().map((m) => ({
|
|
1220
|
+
name: m.name,
|
|
1221
|
+
label: m.label,
|
|
1222
|
+
description: m.description,
|
|
1223
|
+
domain: m.domain,
|
|
1224
|
+
sql: m.sql,
|
|
1225
|
+
type: m.type,
|
|
1226
|
+
table: m.table,
|
|
1227
|
+
tags: m.tags ?? [],
|
|
1228
|
+
owner: m.owner ?? null,
|
|
1229
|
+
metricType: m.metricType ?? null,
|
|
1230
|
+
typeParams: m.typeParams ?? null,
|
|
1231
|
+
filter: m.filter ?? null,
|
|
1232
|
+
source: m.source ?? null,
|
|
1233
|
+
}));
|
|
1234
|
+
const measures = semanticLayer.listMeasures().map((m) => ({
|
|
1235
|
+
name: m.name,
|
|
1236
|
+
label: m.label,
|
|
1237
|
+
description: m.description,
|
|
1238
|
+
domain: m.domain,
|
|
1239
|
+
agg: m.agg,
|
|
1240
|
+
expr: m.expr ?? null,
|
|
1241
|
+
table: m.table,
|
|
1242
|
+
cube: m.cube ?? null,
|
|
1243
|
+
aggTimeDimension: m.aggTimeDimension ?? null,
|
|
1244
|
+
nonAdditiveDimension: m.nonAdditiveDimension ?? null,
|
|
1245
|
+
tags: m.tags ?? [],
|
|
1246
|
+
owner: m.owner ?? null,
|
|
1247
|
+
source: m.source ?? null,
|
|
1248
|
+
}));
|
|
1249
|
+
const dimensions = semanticLayer.listDimensions().map((d) => ({
|
|
1250
|
+
name: d.name,
|
|
1251
|
+
label: d.label,
|
|
1252
|
+
description: d.description,
|
|
1253
|
+
domain: d.domain,
|
|
1254
|
+
sql: d.sql,
|
|
1255
|
+
type: d.type,
|
|
1256
|
+
table: d.table,
|
|
1257
|
+
tags: d.tags ?? [],
|
|
1258
|
+
owner: d.owner ?? null,
|
|
1259
|
+
cube: d.cube ?? null,
|
|
1260
|
+
isTimeDimension: d.isTimeDimension ?? false,
|
|
1261
|
+
typeParams: d.typeParams ?? null,
|
|
1262
|
+
source: d.source ?? null,
|
|
1263
|
+
}));
|
|
1264
|
+
const timeDimensions = semanticLayer.listTimeDimensions().map((d) => ({
|
|
1265
|
+
name: d.name,
|
|
1266
|
+
label: d.label,
|
|
1267
|
+
description: d.description,
|
|
1268
|
+
domain: d.domain,
|
|
1269
|
+
sql: d.sql,
|
|
1270
|
+
type: d.type,
|
|
1271
|
+
table: d.table,
|
|
1272
|
+
cube: d.cube ?? null,
|
|
1273
|
+
granularities: d.granularities ?? [],
|
|
1274
|
+
primaryTime: d.primaryTime ?? false,
|
|
1275
|
+
tags: d.tags ?? [],
|
|
1276
|
+
owner: d.owner ?? null,
|
|
1277
|
+
typeParams: d.typeParams ?? null,
|
|
1278
|
+
source: d.source ?? null,
|
|
1279
|
+
}));
|
|
1280
|
+
const entities = semanticLayer.listEntities().map((e) => ({
|
|
1281
|
+
name: e.name,
|
|
1282
|
+
label: e.label,
|
|
1283
|
+
description: e.description,
|
|
1284
|
+
domain: e.domain,
|
|
1285
|
+
type: e.type,
|
|
1286
|
+
expr: e.expr ?? null,
|
|
1287
|
+
table: e.table,
|
|
1288
|
+
cube: e.cube ?? null,
|
|
1289
|
+
role: e.role ?? null,
|
|
1290
|
+
tags: e.tags ?? [],
|
|
1291
|
+
owner: e.owner ?? null,
|
|
1292
|
+
source: e.source ?? null,
|
|
1293
|
+
}));
|
|
1294
|
+
const hierarchies = semanticLayer.listHierarchies().map((h) => ({
|
|
1295
|
+
name: h.name,
|
|
1296
|
+
label: h.label,
|
|
1297
|
+
description: h.description,
|
|
1298
|
+
domain: h.domain,
|
|
1299
|
+
levels: h.levels.map((l) => ({ name: l.name, label: l.label })),
|
|
1300
|
+
}));
|
|
1301
|
+
const semanticModels = semanticLayer.listSemanticModels().map((m) => ({
|
|
1302
|
+
name: m.name,
|
|
1303
|
+
label: m.label,
|
|
1304
|
+
description: m.description,
|
|
1305
|
+
domain: m.domain,
|
|
1306
|
+
model: m.model ?? null,
|
|
1307
|
+
table: m.table,
|
|
1308
|
+
entities: m.entities,
|
|
1309
|
+
measures: m.measures,
|
|
1310
|
+
dimensions: m.dimensions,
|
|
1311
|
+
timeDimensions: m.timeDimensions,
|
|
1312
|
+
tags: m.tags ?? [],
|
|
1313
|
+
owner: m.owner ?? null,
|
|
1314
|
+
source: m.source ?? null,
|
|
1315
|
+
}));
|
|
1316
|
+
const savedQueries = semanticLayer.listSavedQueries().map((q) => ({
|
|
1317
|
+
name: q.name,
|
|
1318
|
+
label: q.label,
|
|
1319
|
+
description: q.description,
|
|
1320
|
+
domain: q.domain,
|
|
1321
|
+
metrics: q.metrics,
|
|
1322
|
+
dimensions: q.dimensions,
|
|
1323
|
+
timeDimension: q.timeDimension ?? null,
|
|
1324
|
+
granularity: q.granularity ?? null,
|
|
1325
|
+
filters: q.filters ?? null,
|
|
1326
|
+
tags: q.tags ?? [],
|
|
1327
|
+
owner: q.owner ?? null,
|
|
1328
|
+
source: q.source ?? null,
|
|
1329
|
+
}));
|
|
1330
|
+
const provider = projectConfig.semanticLayer?.provider ?? semanticDetectedProvider ?? 'dql';
|
|
1331
|
+
const dbtExecutionReady = provider === 'dbt'
|
|
1332
|
+
? hasDbtSemanticManifest(projectRoot, projectConfig.semanticLayer?.projectPath)
|
|
1333
|
+
: false;
|
|
1334
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1335
|
+
res.end(serializeJSON({
|
|
1336
|
+
available: true,
|
|
1337
|
+
provider,
|
|
1338
|
+
execution: provider === 'dbt'
|
|
1339
|
+
? {
|
|
1340
|
+
engine: 'metricflow',
|
|
1341
|
+
ready: dbtExecutionReady,
|
|
1342
|
+
setup: dbtExecutionReady
|
|
1343
|
+
? null
|
|
1344
|
+
: 'Run `dbt parse` or `dbt build` so target/semantic_manifest.json exists, and install MetricFlow so `mf` is on PATH.',
|
|
1345
|
+
}
|
|
1346
|
+
: { engine: 'native', ready: true, setup: null },
|
|
1347
|
+
errors: semanticLayerErrors,
|
|
1348
|
+
metrics,
|
|
1349
|
+
measures,
|
|
1350
|
+
dimensions,
|
|
1351
|
+
timeDimensions,
|
|
1352
|
+
entities,
|
|
1353
|
+
hierarchies,
|
|
1354
|
+
semanticModels,
|
|
1355
|
+
savedQueries,
|
|
1356
|
+
domains: semanticLayer.listDomains(),
|
|
1357
|
+
tags: semanticLayer.listTags(),
|
|
1358
|
+
favorites: userPrefs.favorites,
|
|
1359
|
+
recentlyUsed: userPrefs.recentlyUsed,
|
|
1360
|
+
lastSyncTime: semanticLastSyncTime,
|
|
1361
|
+
}));
|
|
1362
|
+
return;
|
|
1363
|
+
}
|
|
1364
|
+
if (req.method === 'GET' && path === '/api/semantic-layer/tree') {
|
|
1365
|
+
if (!semanticLayer) {
|
|
1366
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1367
|
+
res.end(serializeJSON({
|
|
1368
|
+
tree: {
|
|
1369
|
+
id: 'provider:dql',
|
|
1370
|
+
label: 'semantic layer',
|
|
1371
|
+
kind: 'provider',
|
|
1372
|
+
count: 0,
|
|
1373
|
+
children: [],
|
|
1374
|
+
},
|
|
1375
|
+
}));
|
|
1376
|
+
return;
|
|
1377
|
+
}
|
|
1378
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1379
|
+
res.end(serializeJSON({
|
|
1380
|
+
tree: buildSemanticTree(semanticLayer, semanticImportManifest),
|
|
1381
|
+
}));
|
|
1382
|
+
return;
|
|
1383
|
+
}
|
|
1384
|
+
if (req.method === 'GET' && path.startsWith('/api/semantic-layer/object/')) {
|
|
1385
|
+
if (!semanticLayer) {
|
|
1386
|
+
res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1387
|
+
res.end(serializeJSON({ error: 'No semantic layer configured.' }));
|
|
1388
|
+
return;
|
|
1389
|
+
}
|
|
1390
|
+
const id = decodeURIComponent(path.slice('/api/semantic-layer/object/'.length));
|
|
1391
|
+
const detail = buildSemanticObjectDetail(semanticLayer, semanticImportManifest, id);
|
|
1392
|
+
if (!detail) {
|
|
1393
|
+
res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1394
|
+
res.end(serializeJSON({ error: `Unknown semantic object: ${id}` }));
|
|
1395
|
+
return;
|
|
1396
|
+
}
|
|
1397
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1398
|
+
res.end(serializeJSON(detail));
|
|
1399
|
+
return;
|
|
1400
|
+
}
|
|
1401
|
+
if (req.method === 'POST' && path === '/api/semantic-layer/import') {
|
|
1402
|
+
try {
|
|
1403
|
+
const body = await readJSON(req);
|
|
1404
|
+
const provider = body.provider;
|
|
1405
|
+
if (provider !== 'dbt' && provider !== 'cubejs' && provider !== 'snowflake') {
|
|
1406
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1407
|
+
res.end(serializeJSON({ error: 'provider must be one of dbt, cubejs, snowflake' }));
|
|
1408
|
+
return;
|
|
1409
|
+
}
|
|
1410
|
+
const sourceConfig = provider === 'snowflake'
|
|
1411
|
+
? {
|
|
1412
|
+
provider,
|
|
1413
|
+
projectPath: body.projectPath ?? projectConfig.semanticLayer?.projectPath,
|
|
1414
|
+
connection: body.connection ?? projectConfig.semanticLayer?.connection,
|
|
1415
|
+
}
|
|
1416
|
+
: {
|
|
1417
|
+
provider,
|
|
1418
|
+
projectPath: typeof body.projectPath === 'string' ? body.projectPath : projectConfig.semanticLayer?.projectPath,
|
|
1419
|
+
repoUrl: typeof body.repoUrl === 'string' ? body.repoUrl : projectConfig.semanticLayer?.repoUrl,
|
|
1420
|
+
branch: typeof body.branch === 'string' ? body.branch : projectConfig.semanticLayer?.branch,
|
|
1421
|
+
subPath: typeof body.subPath === 'string' ? body.subPath : projectConfig.semanticLayer?.subPath,
|
|
1422
|
+
source: body.repoUrl || projectConfig.semanticLayer?.repoUrl
|
|
1423
|
+
? (body.source ?? projectConfig.semanticLayer?.source ?? 'github')
|
|
1424
|
+
: 'local',
|
|
1425
|
+
};
|
|
1426
|
+
const executeQuery = provider === 'snowflake'
|
|
1427
|
+
? async (sql) => {
|
|
1428
|
+
const result = await executor.executeQuery(sql, [], {}, connection);
|
|
1429
|
+
return { rows: result.rows };
|
|
1430
|
+
}
|
|
1431
|
+
: undefined;
|
|
1432
|
+
const importResult = await performSemanticImport({
|
|
1433
|
+
targetProjectRoot: projectRoot,
|
|
1434
|
+
provider,
|
|
1435
|
+
sourceConfig,
|
|
1436
|
+
executeQuery,
|
|
1437
|
+
});
|
|
1438
|
+
// Re-resolve using project's actual semantic config (not hardcoded 'dql')
|
|
1439
|
+
const projSemConfig = loadProjectConfig(projectRoot)?.semanticLayer ?? { provider: 'dql', path: './semantic-layer' };
|
|
1440
|
+
const refreshed = await resolveSemanticLayerAsync(projSemConfig, projectRoot);
|
|
1441
|
+
semanticLayer = refreshed.layer;
|
|
1442
|
+
semanticLayerErrors = refreshed.errors;
|
|
1443
|
+
semanticDetectedProvider = refreshed.detectedProvider ?? 'dql';
|
|
1444
|
+
semanticLastSyncTime = importResult.manifest.importedAt;
|
|
1445
|
+
semanticImportManifest = importResult.manifest;
|
|
1446
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1447
|
+
res.end(serializeJSON(importResult));
|
|
1448
|
+
}
|
|
1449
|
+
catch (error) {
|
|
1450
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1451
|
+
const hint = message.includes('conflict')
|
|
1452
|
+
? 'A file conflict was detected. Remove or rename the conflicting file and retry.'
|
|
1453
|
+
: message.includes('dbt_project.yml')
|
|
1454
|
+
? 'Ensure your dbt project path contains a valid dbt_project.yml file.'
|
|
1455
|
+
: message.includes('query executor')
|
|
1456
|
+
? 'A Snowflake connection is required. Configure one in the Connection panel first.'
|
|
1457
|
+
: 'Check the provider path and ensure the source files are accessible.';
|
|
1458
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1459
|
+
res.end(serializeJSON({ error: message, hint }));
|
|
1460
|
+
}
|
|
1461
|
+
return;
|
|
1462
|
+
}
|
|
1463
|
+
if (req.method === 'POST' && path === '/api/semantic-layer/sync') {
|
|
1464
|
+
try {
|
|
1465
|
+
const executeQuery = semanticImportManifest?.provider === 'snowflake'
|
|
1466
|
+
? async (sql) => {
|
|
1467
|
+
const result = await executor.executeQuery(sql, [], {}, connection);
|
|
1468
|
+
return { rows: result.rows };
|
|
1469
|
+
}
|
|
1470
|
+
: undefined;
|
|
1471
|
+
const importResult = await syncSemanticImport({
|
|
1472
|
+
targetProjectRoot: projectRoot,
|
|
1473
|
+
executeQuery,
|
|
1474
|
+
});
|
|
1475
|
+
// Re-resolve using project's actual semantic config (not hardcoded 'dql')
|
|
1476
|
+
const projSemConfig = loadProjectConfig(projectRoot)?.semanticLayer ?? { provider: 'dql', path: './semantic-layer' };
|
|
1477
|
+
const refreshed = await resolveSemanticLayerAsync(projSemConfig, projectRoot);
|
|
1478
|
+
semanticLayer = refreshed.layer;
|
|
1479
|
+
semanticLayerErrors = refreshed.errors;
|
|
1480
|
+
semanticDetectedProvider = refreshed.detectedProvider ?? 'dql';
|
|
1481
|
+
semanticLastSyncTime = importResult.manifest.importedAt;
|
|
1482
|
+
semanticImportManifest = importResult.manifest;
|
|
1483
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1484
|
+
res.end(serializeJSON(importResult));
|
|
1485
|
+
}
|
|
1486
|
+
catch (error) {
|
|
1487
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1488
|
+
const hint = message.includes('No semantic import manifest')
|
|
1489
|
+
? 'No previous import found. Use the Setup Wizard to import a semantic layer first.'
|
|
1490
|
+
: 'Check the source configuration and retry.';
|
|
1491
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1492
|
+
res.end(serializeJSON({ error: message, hint }));
|
|
1493
|
+
}
|
|
1494
|
+
return;
|
|
1495
|
+
}
|
|
1496
|
+
// ── Semantic layer import preview (dry-run) ──────────────────────────
|
|
1497
|
+
if (req.method === 'POST' && path === '/api/semantic-layer/import-preview') {
|
|
1498
|
+
try {
|
|
1499
|
+
const body = await readJSON(req);
|
|
1500
|
+
const provider = body.provider;
|
|
1501
|
+
if (provider !== 'dbt' && provider !== 'cubejs' && provider !== 'snowflake') {
|
|
1502
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1503
|
+
res.end(serializeJSON({ error: 'provider must be one of dbt, cubejs, snowflake' }));
|
|
1504
|
+
return;
|
|
1505
|
+
}
|
|
1506
|
+
const sourceConfig = provider === 'snowflake'
|
|
1507
|
+
? {
|
|
1508
|
+
provider,
|
|
1509
|
+
projectPath: body.projectPath ?? projectConfig.semanticLayer?.projectPath,
|
|
1510
|
+
connection: body.connection ?? projectConfig.semanticLayer?.connection,
|
|
1511
|
+
}
|
|
1512
|
+
: {
|
|
1513
|
+
provider,
|
|
1514
|
+
projectPath: typeof body.projectPath === 'string' ? body.projectPath : projectConfig.semanticLayer?.projectPath,
|
|
1515
|
+
repoUrl: typeof body.repoUrl === 'string' ? body.repoUrl : projectConfig.semanticLayer?.repoUrl,
|
|
1516
|
+
branch: typeof body.branch === 'string' ? body.branch : projectConfig.semanticLayer?.branch,
|
|
1517
|
+
subPath: typeof body.subPath === 'string' ? body.subPath : projectConfig.semanticLayer?.subPath,
|
|
1518
|
+
source: body.repoUrl || projectConfig.semanticLayer?.repoUrl
|
|
1519
|
+
? (body.source ?? projectConfig.semanticLayer?.source ?? 'github')
|
|
1520
|
+
: 'local',
|
|
1521
|
+
};
|
|
1522
|
+
const executeQuery = provider === 'snowflake'
|
|
1523
|
+
? async (sql) => {
|
|
1524
|
+
const result = await executor.executeQuery(sql, [], {}, connection);
|
|
1525
|
+
return { rows: result.rows };
|
|
1526
|
+
}
|
|
1527
|
+
: undefined;
|
|
1528
|
+
const preview = await previewSemanticImport({
|
|
1529
|
+
targetProjectRoot: projectRoot,
|
|
1530
|
+
provider,
|
|
1531
|
+
sourceConfig,
|
|
1532
|
+
executeQuery,
|
|
1533
|
+
});
|
|
1534
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1535
|
+
res.end(serializeJSON(preview));
|
|
1536
|
+
}
|
|
1537
|
+
catch (error) {
|
|
1538
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1539
|
+
const hint = message.includes('dbt_project.yml')
|
|
1540
|
+
? 'Ensure your dbt project path contains a valid dbt_project.yml file.'
|
|
1541
|
+
: message.includes('model/') || message.includes('schema/')
|
|
1542
|
+
? 'Ensure your Cube.js project has a model/ or schema/ directory.'
|
|
1543
|
+
: message.includes('query executor')
|
|
1544
|
+
? 'A Snowflake connection is required. Configure one in the Connection panel first.'
|
|
1545
|
+
: 'Check the provider path and ensure the source files are accessible.';
|
|
1546
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1547
|
+
res.end(serializeJSON({ error: message, hint }));
|
|
1548
|
+
}
|
|
1549
|
+
return;
|
|
1550
|
+
}
|
|
1551
|
+
// ── Semantic layer sync diff preview ────────────────────────────────
|
|
1552
|
+
if (req.method === 'POST' && path === '/api/semantic-layer/sync-preview') {
|
|
1553
|
+
try {
|
|
1554
|
+
const executeQuery = semanticImportManifest?.provider === 'snowflake'
|
|
1555
|
+
? async (sql) => {
|
|
1556
|
+
const result = await executor.executeQuery(sql, [], {}, connection);
|
|
1557
|
+
return { rows: result.rows };
|
|
1558
|
+
}
|
|
1559
|
+
: undefined;
|
|
1560
|
+
const diff = await computeSyncDiff({
|
|
1561
|
+
targetProjectRoot: projectRoot,
|
|
1562
|
+
executeQuery,
|
|
1563
|
+
});
|
|
1564
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1565
|
+
res.end(serializeJSON(diff));
|
|
1566
|
+
}
|
|
1567
|
+
catch (error) {
|
|
1568
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1569
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
1570
|
+
}
|
|
1571
|
+
return;
|
|
1572
|
+
}
|
|
1573
|
+
if (req.method === 'GET' && path === '/api/semantic-layer/search') {
|
|
1574
|
+
if (!semanticLayer) {
|
|
1575
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1576
|
+
res.end(serializeJSON({ metrics: [], measures: [], dimensions: [], timeDimensions: [], entities: [], hierarchies: [], semanticModels: [], savedQueries: [] }));
|
|
1577
|
+
return;
|
|
1578
|
+
}
|
|
1579
|
+
const q = url.searchParams.get('q') ?? '';
|
|
1580
|
+
const domain = url.searchParams.get('domain') ?? '';
|
|
1581
|
+
const tag = url.searchParams.get('tag') ?? '';
|
|
1582
|
+
const type = url.searchParams.get('type') ?? '';
|
|
1583
|
+
const results = semanticLayer.searchAdvanced(q, {
|
|
1584
|
+
domains: domain ? [domain] : undefined,
|
|
1585
|
+
tags: tag ? [tag] : undefined,
|
|
1586
|
+
types: ['metric', 'measure', 'dimension', 'hierarchy', 'entity', 'semantic_model', 'saved_query'].includes(type) ? [type] : undefined,
|
|
1587
|
+
});
|
|
1588
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1589
|
+
res.end(serializeJSON({
|
|
1590
|
+
metrics: results.metrics.map((m) => ({
|
|
1591
|
+
name: m.name,
|
|
1592
|
+
label: m.label,
|
|
1593
|
+
description: m.description,
|
|
1594
|
+
domain: m.domain,
|
|
1595
|
+
sql: m.sql,
|
|
1596
|
+
type: m.type,
|
|
1597
|
+
table: m.table,
|
|
1598
|
+
tags: m.tags ?? [],
|
|
1599
|
+
owner: m.owner ?? null,
|
|
1600
|
+
})),
|
|
1601
|
+
measures: results.measures.map((m) => ({
|
|
1602
|
+
name: m.name,
|
|
1603
|
+
label: m.label,
|
|
1604
|
+
description: m.description,
|
|
1605
|
+
domain: m.domain,
|
|
1606
|
+
agg: m.agg,
|
|
1607
|
+
expr: m.expr,
|
|
1608
|
+
table: m.table,
|
|
1609
|
+
cube: m.cube,
|
|
1610
|
+
tags: m.tags ?? [],
|
|
1611
|
+
owner: m.owner ?? null,
|
|
1612
|
+
})),
|
|
1613
|
+
dimensions: results.dimensions.map((d) => ({
|
|
1614
|
+
name: d.name,
|
|
1615
|
+
label: d.label,
|
|
1616
|
+
description: d.description,
|
|
1617
|
+
domain: d.domain,
|
|
1618
|
+
sql: d.sql,
|
|
1619
|
+
type: d.type,
|
|
1620
|
+
table: d.table,
|
|
1621
|
+
tags: d.tags ?? [],
|
|
1622
|
+
owner: d.owner ?? null,
|
|
1623
|
+
})),
|
|
1624
|
+
timeDimensions: semanticLayer.listTimeDimensions().filter((d) => results.dimensions.some((dim) => dim.name === d.name)).map((d) => ({
|
|
1625
|
+
name: d.name,
|
|
1626
|
+
label: d.label,
|
|
1627
|
+
description: d.description,
|
|
1628
|
+
domain: d.domain,
|
|
1629
|
+
sql: d.sql,
|
|
1630
|
+
type: d.type,
|
|
1631
|
+
table: d.table,
|
|
1632
|
+
tags: d.tags ?? [],
|
|
1633
|
+
owner: d.owner ?? null,
|
|
1634
|
+
})),
|
|
1635
|
+
entities: results.entities.map((e) => ({
|
|
1636
|
+
name: e.name,
|
|
1637
|
+
label: e.label,
|
|
1638
|
+
description: e.description,
|
|
1639
|
+
domain: e.domain,
|
|
1640
|
+
type: e.type,
|
|
1641
|
+
table: e.table,
|
|
1642
|
+
tags: e.tags ?? [],
|
|
1643
|
+
owner: e.owner ?? null,
|
|
1644
|
+
})),
|
|
1645
|
+
hierarchies: results.hierarchies.map((h) => ({
|
|
1646
|
+
name: h.name,
|
|
1647
|
+
label: h.label,
|
|
1648
|
+
description: h.description,
|
|
1649
|
+
domain: h.domain,
|
|
1650
|
+
levels: h.levels.map((l) => ({ name: l.name, label: l.label })),
|
|
1651
|
+
})),
|
|
1652
|
+
semanticModels: results.semanticModels.map((m) => ({
|
|
1653
|
+
name: m.name,
|
|
1654
|
+
label: m.label,
|
|
1655
|
+
description: m.description,
|
|
1656
|
+
domain: m.domain,
|
|
1657
|
+
table: m.table,
|
|
1658
|
+
measures: m.measures,
|
|
1659
|
+
dimensions: m.dimensions,
|
|
1660
|
+
timeDimensions: m.timeDimensions,
|
|
1661
|
+
})),
|
|
1662
|
+
savedQueries: results.savedQueries.map((q) => ({
|
|
1663
|
+
name: q.name,
|
|
1664
|
+
label: q.label,
|
|
1665
|
+
description: q.description,
|
|
1666
|
+
domain: q.domain,
|
|
1667
|
+
metrics: q.metrics,
|
|
1668
|
+
dimensions: q.dimensions,
|
|
1669
|
+
})),
|
|
1670
|
+
}));
|
|
1671
|
+
return;
|
|
1672
|
+
}
|
|
1673
|
+
if (req.method === 'GET' && path === '/api/semantic-layer/compatible-dims') {
|
|
1674
|
+
if (!semanticLayer) {
|
|
1675
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1676
|
+
res.end(serializeJSON({ dimensions: [] }));
|
|
1677
|
+
return;
|
|
1678
|
+
}
|
|
1679
|
+
const metrics = (url.searchParams.get('metrics') ?? '')
|
|
1680
|
+
.split(',')
|
|
1681
|
+
.map((value) => value.trim())
|
|
1682
|
+
.filter(Boolean);
|
|
1683
|
+
const dimensions = semanticLayer.listCompatibleDimensions(metrics).map((d) => ({
|
|
1684
|
+
name: d.name,
|
|
1685
|
+
label: d.label,
|
|
1686
|
+
description: d.description,
|
|
1687
|
+
domain: d.domain,
|
|
1688
|
+
sql: d.sql,
|
|
1689
|
+
type: d.type,
|
|
1690
|
+
table: d.table,
|
|
1691
|
+
tags: d.tags ?? [],
|
|
1692
|
+
owner: d.owner ?? null,
|
|
1693
|
+
}));
|
|
1694
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1695
|
+
res.end(serializeJSON({ dimensions }));
|
|
1696
|
+
return;
|
|
1697
|
+
}
|
|
1698
|
+
if (req.method === 'GET' && path === '/api/user-prefs/favorites') {
|
|
1699
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1700
|
+
res.end(serializeJSON({ favorites: readUserPrefs(userPrefsPath).favorites }));
|
|
1701
|
+
return;
|
|
1702
|
+
}
|
|
1703
|
+
if (req.method === 'POST' && path === '/api/user-prefs/favorites') {
|
|
1704
|
+
try {
|
|
1705
|
+
const body = await readJSON(req);
|
|
1706
|
+
const prefs = readUserPrefs(userPrefsPath);
|
|
1707
|
+
const name = typeof body.name === 'string' ? body.name.trim() : '';
|
|
1708
|
+
if (name) {
|
|
1709
|
+
prefs.favorites = prefs.favorites.includes(name)
|
|
1710
|
+
? prefs.favorites.filter((item) => item !== name)
|
|
1711
|
+
: [...prefs.favorites, name].sort((a, b) => a.localeCompare(b));
|
|
1712
|
+
writeUserPrefs(userPrefsPath, prefs);
|
|
1713
|
+
}
|
|
1714
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1715
|
+
res.end(serializeJSON({ favorites: prefs.favorites }));
|
|
1716
|
+
}
|
|
1717
|
+
catch (error) {
|
|
1718
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1719
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
1720
|
+
}
|
|
1721
|
+
return;
|
|
1722
|
+
}
|
|
1723
|
+
if (req.method === 'GET' && path === '/api/user-prefs/recent') {
|
|
1724
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1725
|
+
res.end(serializeJSON({ recentlyUsed: readUserPrefs(userPrefsPath).recentlyUsed }));
|
|
1726
|
+
return;
|
|
1727
|
+
}
|
|
1728
|
+
if (req.method === 'POST' && path === '/api/user-prefs/recent') {
|
|
1729
|
+
try {
|
|
1730
|
+
const body = await readJSON(req);
|
|
1731
|
+
const prefs = readUserPrefs(userPrefsPath);
|
|
1732
|
+
const name = typeof body.name === 'string' ? body.name.trim() : '';
|
|
1733
|
+
if (name) {
|
|
1734
|
+
prefs.recentlyUsed = [name, ...prefs.recentlyUsed.filter((item) => item !== name)].slice(0, 12);
|
|
1735
|
+
writeUserPrefs(userPrefsPath, prefs);
|
|
1736
|
+
}
|
|
1737
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1738
|
+
res.end(serializeJSON({ recentlyUsed: prefs.recentlyUsed }));
|
|
1739
|
+
}
|
|
1740
|
+
catch (error) {
|
|
1741
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1742
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
1743
|
+
}
|
|
1744
|
+
return;
|
|
1745
|
+
}
|
|
1746
|
+
// ── Semantic completions for SQL cells ─────────────────────────────────────
|
|
1747
|
+
if (req.method === 'GET' && path === '/api/semantic-completions') {
|
|
1748
|
+
const completions = [];
|
|
1749
|
+
if (semanticLayer) {
|
|
1750
|
+
for (const m of semanticLayer.listMetrics()) {
|
|
1751
|
+
completions.push({
|
|
1752
|
+
type: 'metric',
|
|
1753
|
+
name: m.name,
|
|
1754
|
+
label: m.label,
|
|
1755
|
+
description: m.description ?? '',
|
|
1756
|
+
sql: m.sql,
|
|
1757
|
+
domain: m.domain,
|
|
1758
|
+
tags: m.tags ?? [],
|
|
1759
|
+
});
|
|
1760
|
+
}
|
|
1761
|
+
for (const d of semanticLayer.listDimensions()) {
|
|
1762
|
+
completions.push({
|
|
1763
|
+
type: 'dimension',
|
|
1764
|
+
name: d.name,
|
|
1765
|
+
label: d.label,
|
|
1766
|
+
description: d.description ?? '',
|
|
1767
|
+
sql: d.sql,
|
|
1768
|
+
domain: d.domain,
|
|
1769
|
+
tags: d.tags ?? [],
|
|
1770
|
+
});
|
|
1771
|
+
}
|
|
1772
|
+
}
|
|
1773
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1774
|
+
res.end(serializeJSON({ completions }));
|
|
1775
|
+
return;
|
|
1776
|
+
}
|
|
1777
|
+
// ── end dql-notebook API ──────────────────────────────────────────────────
|
|
1778
|
+
// GET /api/describe-table?table=schema.table — returns columns for a specific table
|
|
1779
|
+
if (req.method === 'GET' && path === '/api/describe-table') {
|
|
1780
|
+
try {
|
|
1781
|
+
const tablePath = url.searchParams.get('table') ?? '';
|
|
1782
|
+
const schemaName = url.searchParams.get('schema') ?? undefined;
|
|
1783
|
+
if (!tablePath) {
|
|
1784
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1785
|
+
res.end(serializeJSON({ error: 'Missing table parameter' }));
|
|
1786
|
+
return;
|
|
1787
|
+
}
|
|
1788
|
+
// Try connector.listColumns() first
|
|
1789
|
+
let columns = [];
|
|
1790
|
+
try {
|
|
1791
|
+
const connector = await executor.getConnector(connection);
|
|
1792
|
+
if (typeof connector.listColumns === 'function') {
|
|
1793
|
+
const rawCols = await connector.listColumns(schemaName, tablePath);
|
|
1794
|
+
columns = rawCols.map((c) => ({ name: c.name, type: c.dataType }));
|
|
1795
|
+
}
|
|
1796
|
+
}
|
|
1797
|
+
catch {
|
|
1798
|
+
// fallback below
|
|
1799
|
+
}
|
|
1800
|
+
// Fallback: DESCRIBE via SQL (works for DuckDB, PG)
|
|
1801
|
+
if (columns.length === 0) {
|
|
1802
|
+
try {
|
|
1803
|
+
const isFile = /\.(csv|parquet|json)$/i.test(tablePath) || tablePath.startsWith('data/');
|
|
1804
|
+
const safePath = tablePath.replace(/'/g, "''");
|
|
1805
|
+
const qualifiedIdentifier = tablePath.split('.').map((p) => `"${p.replace(/"/g, '""')}"`).join('.');
|
|
1806
|
+
const sql = isFile
|
|
1807
|
+
? `DESCRIBE SELECT * FROM read_csv_auto('${safePath}') LIMIT 0`
|
|
1808
|
+
: `DESCRIBE ${qualifiedIdentifier}`;
|
|
1809
|
+
const result = await executor.executeQuery(sql, [], {}, connection);
|
|
1810
|
+
columns = result.rows.map((row) => ({
|
|
1811
|
+
name: String(row['column_name'] ?? row['Field'] ?? ''),
|
|
1812
|
+
type: String(row['column_type'] ?? row['Type'] ?? ''),
|
|
1813
|
+
}));
|
|
1814
|
+
}
|
|
1815
|
+
catch {
|
|
1816
|
+
// empty columns
|
|
1817
|
+
}
|
|
1818
|
+
}
|
|
1819
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1820
|
+
res.end(serializeJSON(columns));
|
|
1821
|
+
}
|
|
1822
|
+
catch (error) {
|
|
1823
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1824
|
+
res.end(serializeJSON({ error: String(error) }));
|
|
1825
|
+
}
|
|
1826
|
+
return;
|
|
1827
|
+
}
|
|
1828
|
+
if (req.method === 'POST' && path === '/api/llm/run') {
|
|
1829
|
+
const body = await readJSON(req).catch(() => null);
|
|
1830
|
+
if (!body || typeof body !== 'object') {
|
|
1831
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1832
|
+
res.end(serializeJSON({ error: 'Invalid JSON body' }));
|
|
1833
|
+
return;
|
|
1834
|
+
}
|
|
1835
|
+
const { provider, messages, upstream } = body;
|
|
1836
|
+
const runner = isLLMProviderId(provider) ? getLLMRunner(provider) : null;
|
|
1837
|
+
if (!runner) {
|
|
1838
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1839
|
+
res.end(serializeJSON({ error: `Unknown provider: ${provider}` }));
|
|
1840
|
+
return;
|
|
1841
|
+
}
|
|
1842
|
+
if (!Array.isArray(messages) || messages.length === 0) {
|
|
1843
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1844
|
+
res.end(serializeJSON({ error: 'messages[] required' }));
|
|
1845
|
+
return;
|
|
1846
|
+
}
|
|
1847
|
+
res.writeHead(200, {
|
|
1848
|
+
'Content-Type': 'text/event-stream',
|
|
1849
|
+
'Cache-Control': 'no-cache, no-transform',
|
|
1850
|
+
Connection: 'keep-alive',
|
|
1851
|
+
});
|
|
1852
|
+
const controller = new AbortController();
|
|
1853
|
+
req.on('close', () => controller.abort());
|
|
1854
|
+
const emit = (turn) => { res.write(`data: ${JSON.stringify(turn)}\n\n`); };
|
|
1855
|
+
try {
|
|
1856
|
+
await runner.run({ provider: provider, messages, upstream, projectRoot }, emit, controller.signal);
|
|
1857
|
+
}
|
|
1858
|
+
catch (err) {
|
|
1859
|
+
emit({ kind: 'error', message: err instanceof Error ? err.message : String(err) });
|
|
1860
|
+
}
|
|
1861
|
+
res.end();
|
|
1862
|
+
return;
|
|
1863
|
+
}
|
|
1864
|
+
if (req.method === 'POST' && path === '/api/query') {
|
|
1865
|
+
try {
|
|
1866
|
+
const body = await readJSON(req);
|
|
1867
|
+
if (typeof body.sql !== 'string' || body.sql.trim().length === 0) {
|
|
1868
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1869
|
+
res.end(serializeJSON({ columns: [], rows: [], error: 'Missing SQL in request body.' }));
|
|
1870
|
+
return;
|
|
1871
|
+
}
|
|
1872
|
+
const semantic = prepareSemanticSql(body.sql, semanticLayer);
|
|
1873
|
+
if (semantic.unresolvedRefs.length > 0) {
|
|
1874
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1875
|
+
res.end(serializeJSON({
|
|
1876
|
+
columns: [],
|
|
1877
|
+
rows: [],
|
|
1878
|
+
error: `Unknown semantic reference${semantic.unresolvedRefs.length > 1 ? 's' : ''}: ${semantic.unresolvedRefs.join(', ')}`,
|
|
1879
|
+
code: 'semantic_ref',
|
|
1880
|
+
unresolvedRefs: semantic.unresolvedRefs,
|
|
1881
|
+
}));
|
|
1882
|
+
return;
|
|
1883
|
+
}
|
|
1884
|
+
const prepared = prepareLocalExecution(semantic.sql, isConnectionConfig(body.connection) ? body.connection : connection, projectRoot, projectConfig);
|
|
1885
|
+
const app = loadRuntimeApp(projectRoot, typeof body.appId === 'string' ? body.appId : activePersonaAppId());
|
|
1886
|
+
const domain = typeof body.domain === 'string' ? body.domain : app?.domain;
|
|
1887
|
+
assertAppAccess({ app, domain, level: 'execute' });
|
|
1888
|
+
const result = await executor.executeQuery(prepared.sql, Array.isArray(body.sqlParams) ? body.sqlParams : [], runtimeVariables(body.variables && typeof body.variables === 'object' ? body.variables : {}), prepared.connection);
|
|
1889
|
+
const payload = serializeJSON(normalizeQueryResult(result, semantic.semanticRefs));
|
|
1890
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1891
|
+
res.end(payload);
|
|
1892
|
+
}
|
|
1893
|
+
catch (error) {
|
|
1894
|
+
if (res.headersSent || res.writableEnded) {
|
|
1895
|
+
res.end();
|
|
1896
|
+
return;
|
|
1897
|
+
}
|
|
1898
|
+
if (error instanceof DQLAccessDeniedError) {
|
|
1899
|
+
res.writeHead(403, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1900
|
+
res.end(serializeJSON({
|
|
1901
|
+
columns: [],
|
|
1902
|
+
rows: [],
|
|
1903
|
+
error: error.message,
|
|
1904
|
+
code: 'unauthorized',
|
|
1905
|
+
}));
|
|
1906
|
+
return;
|
|
1907
|
+
}
|
|
1908
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1909
|
+
res.end(serializeJSON({
|
|
1910
|
+
columns: [],
|
|
1911
|
+
rows: [],
|
|
1912
|
+
error: error instanceof Error ? error.message : String(error),
|
|
1913
|
+
}));
|
|
1914
|
+
}
|
|
1915
|
+
return;
|
|
1916
|
+
}
|
|
1917
|
+
// Semantic layer query endpoint: compose SQL from metrics/dimensions
|
|
1918
|
+
if (req.method === 'POST' && path === '/api/semantic-query') {
|
|
1919
|
+
try {
|
|
1920
|
+
if (!semanticLayer) {
|
|
1921
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1922
|
+
res.end(serializeJSON({ error: 'No semantic layer configured. Add YAML files to semantic-layer/ directory.' }));
|
|
1923
|
+
return;
|
|
1924
|
+
}
|
|
1925
|
+
const body = await readJSON(req);
|
|
1926
|
+
const { metrics = [], dimensions = [], filters = [], limit, timeDimension, orderBy, savedQuery, engine } = body;
|
|
1927
|
+
// Resolve which connection to use — request can override default
|
|
1928
|
+
const targetConnection = isConnectionConfig(body.connection) ? body.connection : connection;
|
|
1929
|
+
const driver = targetConnection.driver;
|
|
1930
|
+
// Build table mapping: resolve semantic model names to actual DB table names
|
|
1931
|
+
let tableMapping;
|
|
1932
|
+
try {
|
|
1933
|
+
const tablesResult = await executor.executeQuery(`SELECT table_schema, table_name FROM information_schema.tables WHERE table_schema NOT IN ('information_schema', 'pg_catalog')`, [], {}, targetConnection);
|
|
1934
|
+
const dbTableNames = new Set();
|
|
1935
|
+
const schemaQualified = new Map();
|
|
1936
|
+
for (const row of tablesResult.rows) {
|
|
1937
|
+
const schema = String(row['table_schema'] ?? '');
|
|
1938
|
+
const name = String(row['table_name'] ?? '');
|
|
1939
|
+
dbTableNames.add(name);
|
|
1940
|
+
schemaQualified.set(name, schema ? `${schema}.${name}` : name);
|
|
1941
|
+
}
|
|
1942
|
+
// For each table in the semantic layer, map to qualified name if it exists
|
|
1943
|
+
const allSemanticTables = new Set();
|
|
1944
|
+
for (const m of semanticLayer.listMetrics())
|
|
1945
|
+
allSemanticTables.add(m.table);
|
|
1946
|
+
for (const d of semanticLayer.listDimensions())
|
|
1947
|
+
allSemanticTables.add(d.table);
|
|
1948
|
+
tableMapping = {};
|
|
1949
|
+
for (const semTable of allSemanticTables) {
|
|
1950
|
+
if (dbTableNames.has(semTable) && schemaQualified.has(semTable)) {
|
|
1951
|
+
tableMapping[semTable] = schemaQualified.get(semTable);
|
|
1952
|
+
}
|
|
1953
|
+
}
|
|
1954
|
+
if (Object.keys(tableMapping).length === 0)
|
|
1955
|
+
tableMapping = undefined;
|
|
1956
|
+
}
|
|
1957
|
+
catch {
|
|
1958
|
+
// Non-fatal: proceed without table mapping
|
|
1959
|
+
}
|
|
1960
|
+
const composed = composeRuntimeSemanticQuery({
|
|
1961
|
+
metrics,
|
|
1962
|
+
dimensions,
|
|
1963
|
+
filters,
|
|
1964
|
+
limit,
|
|
1965
|
+
timeDimension,
|
|
1966
|
+
orderBy,
|
|
1967
|
+
savedQuery,
|
|
1968
|
+
engine,
|
|
1969
|
+
}, semanticLayer, {
|
|
1970
|
+
projectRoot,
|
|
1971
|
+
projectConfig,
|
|
1972
|
+
detectedProvider: semanticDetectedProvider,
|
|
1973
|
+
driver,
|
|
1974
|
+
tableMapping,
|
|
1975
|
+
});
|
|
1976
|
+
if (!composed) {
|
|
1977
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1978
|
+
res.end(serializeJSON({ error: `Could not compose query for metrics: [${metrics.join(', ')}]` }));
|
|
1979
|
+
return;
|
|
1980
|
+
}
|
|
1981
|
+
// Execute the composed SQL against the resolved connection
|
|
1982
|
+
const prepared = prepareLocalExecution(composed.sql, targetConnection, projectRoot, projectConfig);
|
|
1983
|
+
const result = await executor.executeQuery(prepared.sql, [], {}, prepared.connection);
|
|
1984
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1985
|
+
res.end(serializeJSON({
|
|
1986
|
+
sql: composed.sql,
|
|
1987
|
+
tables: composed.tables,
|
|
1988
|
+
joins: composed.joins,
|
|
1989
|
+
engine: composed.engine,
|
|
1990
|
+
result: normalizeQueryResult(result),
|
|
1991
|
+
}));
|
|
1992
|
+
}
|
|
1993
|
+
catch (error) {
|
|
1994
|
+
if (error instanceof DQLAccessDeniedError) {
|
|
1995
|
+
res.writeHead(403, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
1996
|
+
res.end(serializeJSON({ error: error.message, code: 'unauthorized' }));
|
|
1997
|
+
return;
|
|
1998
|
+
}
|
|
1999
|
+
const status = error instanceof MetricFlowUnavailableError ? 400 : 500;
|
|
2000
|
+
res.writeHead(status, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2001
|
+
res.end(serializeJSON({
|
|
2002
|
+
error: error instanceof Error ? error.message : String(error),
|
|
2003
|
+
code: error instanceof MetricFlowUnavailableError ? 'metricflow_unavailable' : undefined,
|
|
2004
|
+
hint: error instanceof MetricFlowUnavailableError
|
|
2005
|
+
? 'Install dbt Semantic Layer dependencies, run dbt parse/build to create target/semantic_manifest.json, then retry.'
|
|
2006
|
+
: undefined,
|
|
2007
|
+
}));
|
|
2008
|
+
}
|
|
2009
|
+
return;
|
|
2010
|
+
}
|
|
2011
|
+
if (req.method === 'POST' && path === '/api/semantic-builder/preview') {
|
|
2012
|
+
try {
|
|
2013
|
+
if (!semanticLayer) {
|
|
2014
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2015
|
+
res.end(serializeJSON({ error: 'No semantic layer configured.' }));
|
|
2016
|
+
return;
|
|
2017
|
+
}
|
|
2018
|
+
const body = await readJSON(req);
|
|
2019
|
+
const { metrics = [], dimensions = [], filters = [], limit, timeDimension, orderBy, savedQuery, engine } = body;
|
|
2020
|
+
const targetConnection = isConnectionConfig(body.connection) ? body.connection : connection;
|
|
2021
|
+
const driver = targetConnection.driver;
|
|
2022
|
+
let tableMapping;
|
|
2023
|
+
try {
|
|
2024
|
+
const tablesResult = await executor.executeQuery(`SELECT table_schema, table_name FROM information_schema.tables WHERE table_schema NOT IN ('information_schema', 'pg_catalog')`, [], {}, targetConnection);
|
|
2025
|
+
const schemaQualified = new Map();
|
|
2026
|
+
for (const row of tablesResult.rows) {
|
|
2027
|
+
const schema = String(row['table_schema'] ?? '');
|
|
2028
|
+
const name = String(row['table_name'] ?? '');
|
|
2029
|
+
schemaQualified.set(name, schema ? `${schema}.${name}` : name);
|
|
2030
|
+
}
|
|
2031
|
+
tableMapping = {};
|
|
2032
|
+
for (const metric of semanticLayer.listMetrics()) {
|
|
2033
|
+
if (schemaQualified.has(metric.table))
|
|
2034
|
+
tableMapping[metric.table] = schemaQualified.get(metric.table);
|
|
2035
|
+
}
|
|
2036
|
+
for (const dimension of semanticLayer.listDimensions()) {
|
|
2037
|
+
if (schemaQualified.has(dimension.table))
|
|
2038
|
+
tableMapping[dimension.table] = schemaQualified.get(dimension.table);
|
|
2039
|
+
}
|
|
2040
|
+
if (Object.keys(tableMapping).length === 0)
|
|
2041
|
+
tableMapping = undefined;
|
|
2042
|
+
}
|
|
2043
|
+
catch {
|
|
2044
|
+
tableMapping = undefined;
|
|
2045
|
+
}
|
|
2046
|
+
const composed = composeRuntimeSemanticQuery({
|
|
2047
|
+
metrics,
|
|
2048
|
+
dimensions,
|
|
2049
|
+
filters,
|
|
2050
|
+
limit,
|
|
2051
|
+
timeDimension,
|
|
2052
|
+
orderBy,
|
|
2053
|
+
savedQuery,
|
|
2054
|
+
engine,
|
|
2055
|
+
}, semanticLayer, {
|
|
2056
|
+
projectRoot,
|
|
2057
|
+
projectConfig,
|
|
2058
|
+
detectedProvider: semanticDetectedProvider,
|
|
2059
|
+
driver,
|
|
2060
|
+
tableMapping,
|
|
2061
|
+
});
|
|
2062
|
+
if (!composed) {
|
|
2063
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2064
|
+
res.end(serializeJSON({ error: 'Could not compose semantic block preview SQL.' }));
|
|
2065
|
+
return;
|
|
2066
|
+
}
|
|
2067
|
+
const prepared = prepareLocalExecution(composed.sql, targetConnection, projectRoot, projectConfig);
|
|
2068
|
+
const result = await executor.executeQuery(prepared.sql, [], {}, prepared.connection);
|
|
2069
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2070
|
+
res.end(serializeJSON({
|
|
2071
|
+
sql: composed.sql,
|
|
2072
|
+
joins: composed.joins,
|
|
2073
|
+
tables: composed.tables,
|
|
2074
|
+
engine: composed.engine,
|
|
2075
|
+
result: normalizeQueryResult(result),
|
|
2076
|
+
}));
|
|
2077
|
+
}
|
|
2078
|
+
catch (error) {
|
|
2079
|
+
const status = error instanceof MetricFlowUnavailableError ? 400 : 500;
|
|
2080
|
+
res.writeHead(status, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2081
|
+
res.end(serializeJSON({
|
|
2082
|
+
error: error instanceof Error ? error.message : String(error),
|
|
2083
|
+
code: error instanceof MetricFlowUnavailableError ? 'metricflow_unavailable' : undefined,
|
|
2084
|
+
hint: error instanceof MetricFlowUnavailableError
|
|
2085
|
+
? 'Install dbt Semantic Layer dependencies, run dbt parse/build to create target/semantic_manifest.json, then retry.'
|
|
2086
|
+
: undefined,
|
|
2087
|
+
}));
|
|
2088
|
+
}
|
|
2089
|
+
return;
|
|
2090
|
+
}
|
|
2091
|
+
if (req.method === 'POST' && path === '/api/semantic-builder/save') {
|
|
2092
|
+
try {
|
|
2093
|
+
if (!semanticLayer) {
|
|
2094
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2095
|
+
res.end(serializeJSON({ error: 'No semantic layer configured.' }));
|
|
2096
|
+
return;
|
|
2097
|
+
}
|
|
2098
|
+
const body = await readJSON(req);
|
|
2099
|
+
const { name, domain, description, owner, tags, metrics = [], dimensions = [], timeDimension, filters = [], chart = 'table', blockType = 'semantic', engine, } = body;
|
|
2100
|
+
if (!name || metrics.length === 0) {
|
|
2101
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2102
|
+
res.end(serializeJSON({ error: 'name and at least one metric are required.' }));
|
|
2103
|
+
return;
|
|
2104
|
+
}
|
|
2105
|
+
const targetConnection = isConnectionConfig(body.connection) ? body.connection : connection;
|
|
2106
|
+
const composed = composeRuntimeSemanticQuery({
|
|
2107
|
+
metrics,
|
|
2108
|
+
dimensions,
|
|
2109
|
+
filters,
|
|
2110
|
+
timeDimension,
|
|
2111
|
+
engine,
|
|
2112
|
+
}, semanticLayer, {
|
|
2113
|
+
projectRoot,
|
|
2114
|
+
projectConfig,
|
|
2115
|
+
detectedProvider: semanticDetectedProvider,
|
|
2116
|
+
driver: targetConnection.driver,
|
|
2117
|
+
});
|
|
2118
|
+
if (!composed) {
|
|
2119
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2120
|
+
res.end(serializeJSON({ error: 'Could not compose semantic block SQL.' }));
|
|
2121
|
+
return;
|
|
2122
|
+
}
|
|
2123
|
+
const created = createSemanticBuilderBlock(projectRoot, {
|
|
2124
|
+
name,
|
|
2125
|
+
domain,
|
|
2126
|
+
description,
|
|
2127
|
+
owner,
|
|
2128
|
+
tags,
|
|
2129
|
+
metrics,
|
|
2130
|
+
dimensions,
|
|
2131
|
+
timeDimension,
|
|
2132
|
+
chart,
|
|
2133
|
+
blockType,
|
|
2134
|
+
sql: composed.sql,
|
|
2135
|
+
tables: composed.tables,
|
|
2136
|
+
provider: semanticImportManifest?.provider ?? semanticDetectedProvider ?? 'dql',
|
|
2137
|
+
});
|
|
2138
|
+
res.writeHead(201, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2139
|
+
res.end(serializeJSON(created));
|
|
2140
|
+
}
|
|
2141
|
+
catch (error) {
|
|
2142
|
+
if (error instanceof Error && error.message === 'BLOCK_EXISTS') {
|
|
2143
|
+
res.writeHead(409, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2144
|
+
res.end(serializeJSON({ error: 'Block already exists' }));
|
|
2145
|
+
return;
|
|
2146
|
+
}
|
|
2147
|
+
const status = error instanceof MetricFlowUnavailableError ? 400 : 500;
|
|
2148
|
+
res.writeHead(status, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2149
|
+
res.end(serializeJSON({
|
|
2150
|
+
error: error instanceof Error ? error.message : String(error),
|
|
2151
|
+
code: error instanceof MetricFlowUnavailableError ? 'metricflow_unavailable' : undefined,
|
|
2152
|
+
hint: error instanceof MetricFlowUnavailableError
|
|
2153
|
+
? 'Install dbt Semantic Layer dependencies, run dbt parse/build to create target/semantic_manifest.json, then retry.'
|
|
2154
|
+
: undefined,
|
|
2155
|
+
}));
|
|
2156
|
+
}
|
|
2157
|
+
return;
|
|
2158
|
+
}
|
|
2159
|
+
if (req.method === 'POST' && path === '/api/test-connection') {
|
|
2160
|
+
try {
|
|
2161
|
+
const body = await readJSON(req);
|
|
2162
|
+
const target = normalizeProjectConnection(isConnectionConfig(body.connection) ? body.connection : connection, projectRoot);
|
|
2163
|
+
const connector = await executor.getConnector(target);
|
|
2164
|
+
const ok = await connector.ping();
|
|
2165
|
+
const driver = target.driver ?? 'unknown';
|
|
2166
|
+
res.writeHead(ok ? 200 : 400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2167
|
+
res.end(serializeJSON({
|
|
2168
|
+
ok,
|
|
2169
|
+
message: ok ? `Connected to ${driver} successfully` : `Connection to ${driver} failed`,
|
|
2170
|
+
}));
|
|
2171
|
+
}
|
|
2172
|
+
catch (error) {
|
|
2173
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2174
|
+
res.end(serializeJSON({
|
|
2175
|
+
ok: false,
|
|
2176
|
+
message: error instanceof Error ? error.message : String(error),
|
|
2177
|
+
}));
|
|
2178
|
+
}
|
|
2179
|
+
return;
|
|
2180
|
+
}
|
|
2181
|
+
// ---- Lineage API ----
|
|
2182
|
+
if (req.method === 'GET' && path === '/api/lineage') {
|
|
2183
|
+
try {
|
|
2184
|
+
const graph = buildProjectLineageGraph(projectRoot, semanticLayer);
|
|
2185
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2186
|
+
res.end(serializeJSON(graph.toJSON()));
|
|
2187
|
+
}
|
|
2188
|
+
catch (error) {
|
|
2189
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2190
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
2191
|
+
}
|
|
2192
|
+
return;
|
|
2193
|
+
}
|
|
2194
|
+
if (req.method === 'GET' && path === '/api/lineage/search') {
|
|
2195
|
+
const term = url.searchParams.get('q') ?? '';
|
|
2196
|
+
try {
|
|
2197
|
+
const graph = buildProjectLineageGraph(projectRoot, semanticLayer);
|
|
2198
|
+
const result = queryLineage(graph, { search: term });
|
|
2199
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2200
|
+
res.end(serializeJSON({ matches: result.matches ?? [] }));
|
|
2201
|
+
}
|
|
2202
|
+
catch (error) {
|
|
2203
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2204
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
2205
|
+
}
|
|
2206
|
+
return;
|
|
2207
|
+
}
|
|
2208
|
+
if (req.method === 'GET' && path === '/api/lineage/query') {
|
|
2209
|
+
try {
|
|
2210
|
+
const graph = buildProjectLineageGraph(projectRoot, semanticLayer);
|
|
2211
|
+
const types = url.searchParams.get('types')
|
|
2212
|
+
?.split(',')
|
|
2213
|
+
.map((value) => value.trim())
|
|
2214
|
+
.filter(Boolean);
|
|
2215
|
+
const upstreamDepthParam = url.searchParams.get('upstreamDepth');
|
|
2216
|
+
const downstreamDepthParam = url.searchParams.get('downstreamDepth');
|
|
2217
|
+
const result = queryLineage(graph, {
|
|
2218
|
+
focus: url.searchParams.get('focus') ?? undefined,
|
|
2219
|
+
search: url.searchParams.get('search') ?? undefined,
|
|
2220
|
+
types,
|
|
2221
|
+
domain: url.searchParams.get('domain') ?? undefined,
|
|
2222
|
+
upstreamDepth: upstreamDepthParam ? Number(upstreamDepthParam) : undefined,
|
|
2223
|
+
downstreamDepth: downstreamDepthParam ? Number(downstreamDepthParam) : undefined,
|
|
2224
|
+
});
|
|
2225
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2226
|
+
res.end(serializeJSON(result));
|
|
2227
|
+
}
|
|
2228
|
+
catch (error) {
|
|
2229
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2230
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
2231
|
+
}
|
|
2232
|
+
return;
|
|
2233
|
+
}
|
|
2234
|
+
if (req.method === 'GET' && path.startsWith('/api/lineage/node/')) {
|
|
2235
|
+
const rawNodeId = decodeURIComponent(path.slice('/api/lineage/node/'.length));
|
|
2236
|
+
try {
|
|
2237
|
+
const graph = buildProjectLineageGraph(projectRoot, semanticLayer);
|
|
2238
|
+
const node = resolveLineageNode(graph, rawNodeId);
|
|
2239
|
+
if (!node) {
|
|
2240
|
+
res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2241
|
+
res.end(serializeJSON({ error: `Lineage node "${rawNodeId}" not found` }));
|
|
2242
|
+
return;
|
|
2243
|
+
}
|
|
2244
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2245
|
+
res.end(serializeJSON({
|
|
2246
|
+
node,
|
|
2247
|
+
incoming: graph.getIncomingEdges(node.id).map((edge) => ({
|
|
2248
|
+
edge,
|
|
2249
|
+
node: graph.getNode(edge.source),
|
|
2250
|
+
})),
|
|
2251
|
+
outgoing: graph.getOutgoingEdges(node.id).map((edge) => ({
|
|
2252
|
+
edge,
|
|
2253
|
+
node: graph.getNode(edge.target),
|
|
2254
|
+
})),
|
|
2255
|
+
}));
|
|
2256
|
+
}
|
|
2257
|
+
catch (error) {
|
|
2258
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2259
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
2260
|
+
}
|
|
2261
|
+
return;
|
|
2262
|
+
}
|
|
2263
|
+
if (req.method === 'GET' && path.startsWith('/api/lineage/domain/')) {
|
|
2264
|
+
const domain = decodeURIComponent(path.slice('/api/lineage/domain/'.length));
|
|
2265
|
+
try {
|
|
2266
|
+
const graph = buildProjectLineageGraph(projectRoot, semanticLayer);
|
|
2267
|
+
const overview = getDomainTrustOverview(graph, domain);
|
|
2268
|
+
const nodes = graph.getNodesByDomain(domain);
|
|
2269
|
+
const flows = detectDomainFlows(graph);
|
|
2270
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2271
|
+
res.end(serializeJSON({
|
|
2272
|
+
domain,
|
|
2273
|
+
overview,
|
|
2274
|
+
nodes,
|
|
2275
|
+
inFlows: flows.filter((f) => f.to === domain),
|
|
2276
|
+
outFlows: flows.filter((f) => f.from === domain),
|
|
2277
|
+
}));
|
|
2278
|
+
}
|
|
2279
|
+
catch (error) {
|
|
2280
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2281
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
2282
|
+
}
|
|
2283
|
+
return;
|
|
2284
|
+
}
|
|
2285
|
+
if (req.method === 'GET' && path.startsWith('/api/lineage/impact/')) {
|
|
2286
|
+
const blockName = decodeURIComponent(path.slice('/api/lineage/impact/'.length));
|
|
2287
|
+
try {
|
|
2288
|
+
const graph = buildProjectLineageGraph(projectRoot, semanticLayer);
|
|
2289
|
+
const nodeId = `block:${blockName}`;
|
|
2290
|
+
if (!graph.getNode(nodeId)) {
|
|
2291
|
+
res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2292
|
+
res.end(serializeJSON({ error: `Block "${blockName}" not found` }));
|
|
2293
|
+
return;
|
|
2294
|
+
}
|
|
2295
|
+
const impact = analyzeImpact(graph, nodeId);
|
|
2296
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2297
|
+
res.end(serializeJSON(impact));
|
|
2298
|
+
}
|
|
2299
|
+
catch (error) {
|
|
2300
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2301
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
2302
|
+
}
|
|
2303
|
+
return;
|
|
2304
|
+
}
|
|
2305
|
+
if (req.method === 'GET' && path.startsWith('/api/lineage/block/')) {
|
|
2306
|
+
const blockName = decodeURIComponent(path.slice('/api/lineage/block/'.length));
|
|
2307
|
+
try {
|
|
2308
|
+
const graph = buildProjectLineageGraph(projectRoot, semanticLayer);
|
|
2309
|
+
const nodeId = `block:${blockName}`;
|
|
2310
|
+
const node = graph.getNode(nodeId);
|
|
2311
|
+
if (!node) {
|
|
2312
|
+
res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2313
|
+
res.end(serializeJSON({ error: `Block "${blockName}" not found` }));
|
|
2314
|
+
return;
|
|
2315
|
+
}
|
|
2316
|
+
const ancestors = graph.ancestors(nodeId);
|
|
2317
|
+
const descendants = graph.descendants(nodeId);
|
|
2318
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2319
|
+
res.end(serializeJSON({ node, ancestors, descendants }));
|
|
2320
|
+
}
|
|
2321
|
+
catch (error) {
|
|
2322
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2323
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
2324
|
+
}
|
|
2325
|
+
return;
|
|
2326
|
+
}
|
|
2327
|
+
if (req.method === 'GET' && path.startsWith('/api/lineage/paths/')) {
|
|
2328
|
+
const rawNodeId = decodeURIComponent(path.slice('/api/lineage/paths/'.length));
|
|
2329
|
+
try {
|
|
2330
|
+
const graph = buildProjectLineageGraph(projectRoot, semanticLayer);
|
|
2331
|
+
const maxDepth = Number(url.searchParams.get('maxDepth') ?? '10') || 10;
|
|
2332
|
+
const maxPaths = Number(url.searchParams.get('maxPaths') ?? '20') || 20;
|
|
2333
|
+
const result = queryCompleteLineagePaths(graph, rawNodeId, { maxDepth, maxPaths });
|
|
2334
|
+
if (!result) {
|
|
2335
|
+
res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2336
|
+
res.end(serializeJSON({ error: `Node "${rawNodeId}" not found` }));
|
|
2337
|
+
return;
|
|
2338
|
+
}
|
|
2339
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2340
|
+
res.end(serializeJSON(result));
|
|
2341
|
+
}
|
|
2342
|
+
catch (error) {
|
|
2343
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2344
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
2345
|
+
}
|
|
2346
|
+
return;
|
|
2347
|
+
}
|
|
2348
|
+
if (req.method === 'GET' && path === '/api/lineage/trust-chain') {
|
|
2349
|
+
const from = url.searchParams.get('from');
|
|
2350
|
+
const to = url.searchParams.get('to');
|
|
2351
|
+
if (!from || !to) {
|
|
2352
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2353
|
+
res.end(serializeJSON({ error: 'Missing "from" and "to" query parameters' }));
|
|
2354
|
+
return;
|
|
2355
|
+
}
|
|
2356
|
+
try {
|
|
2357
|
+
const graph = buildProjectLineageGraph(projectRoot, semanticLayer);
|
|
2358
|
+
const chain = buildTrustChain(graph, `block:${from}`, `block:${to}`);
|
|
2359
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2360
|
+
res.end(serializeJSON(chain ?? { error: 'No path found' }));
|
|
2361
|
+
}
|
|
2362
|
+
catch (error) {
|
|
2363
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2364
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
2365
|
+
}
|
|
2366
|
+
return;
|
|
2367
|
+
}
|
|
2368
|
+
if (req.method === 'GET' && path === '/api/notebook/bootstrap') {
|
|
2369
|
+
const welcomeNotebook = resolveNotebook(projectRoot, projectConfig.project ?? 'DQL Project');
|
|
2370
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2371
|
+
res.end(serializeJSON({
|
|
2372
|
+
projectRoot,
|
|
2373
|
+
project: projectConfig.project ?? 'DQL Project',
|
|
2374
|
+
defaultConnection: projectConfig.defaultConnection ?? connection,
|
|
2375
|
+
connectorForms: getConnectorFormSchemas(),
|
|
2376
|
+
files: listProjectFiles(projectRoot),
|
|
2377
|
+
notebook: welcomeNotebook,
|
|
2378
|
+
}));
|
|
2379
|
+
return;
|
|
2380
|
+
}
|
|
2381
|
+
if (req.method === 'GET' && path === '/api/notebook/file') {
|
|
2382
|
+
const relativePath = url.searchParams.get('path');
|
|
2383
|
+
if (!relativePath) {
|
|
2384
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2385
|
+
res.end(serializeJSON({ error: 'Missing file path.' }));
|
|
2386
|
+
return;
|
|
2387
|
+
}
|
|
2388
|
+
const filePath = safeJoin(projectRoot, relativePath);
|
|
2389
|
+
if (!filePath || !existsSync(filePath) || statSync(filePath).isDirectory()) {
|
|
2390
|
+
res.writeHead(404, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2391
|
+
res.end(serializeJSON({ error: `File not found: ${relativePath}` }));
|
|
2392
|
+
return;
|
|
2393
|
+
}
|
|
2394
|
+
res.writeHead(200, { 'Content-Type': contentTypeFor(filePath) });
|
|
2395
|
+
res.end(readFileSync(filePath));
|
|
2396
|
+
return;
|
|
2397
|
+
}
|
|
2398
|
+
if (req.method === 'POST' && path === '/api/notebook/execute') {
|
|
2399
|
+
try {
|
|
2400
|
+
const body = await readJSON(req);
|
|
2401
|
+
const cell = normalizeNotebookCell(body.cell);
|
|
2402
|
+
if (!cell) {
|
|
2403
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2404
|
+
res.end(serializeJSON({ error: 'Missing notebook cell payload.' }));
|
|
2405
|
+
return;
|
|
2406
|
+
}
|
|
2407
|
+
const cellConnection = isConnectionConfig(body.connection) ? body.connection : connection;
|
|
2408
|
+
const plan = buildExecutionPlan(cell, { semanticLayer, driver: cellConnection.driver });
|
|
2409
|
+
if (!plan) {
|
|
2410
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2411
|
+
res.end(serializeJSON({ cellType: cell.type, result: null }));
|
|
2412
|
+
return;
|
|
2413
|
+
}
|
|
2414
|
+
const prepared = prepareLocalExecution(plan.sql, isConnectionConfig(body.connection) ? body.connection : connection, projectRoot, projectConfig);
|
|
2415
|
+
const app = loadRuntimeApp(projectRoot, typeof body.appId === 'string' ? body.appId : activePersonaAppId());
|
|
2416
|
+
assertAppAccess({ app, domain: app?.domain, level: 'execute' });
|
|
2417
|
+
const rawResult = await executor.executeQuery(prepared.sql, plan.sqlParams, runtimeVariables(plan.variables), prepared.connection);
|
|
2418
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2419
|
+
res.end(serializeJSON({
|
|
2420
|
+
cellType: cell.type,
|
|
2421
|
+
title: plan.title,
|
|
2422
|
+
chartConfig: plan.chartConfig,
|
|
2423
|
+
tests: plan.tests,
|
|
2424
|
+
result: normalizeQueryResult(rawResult),
|
|
2425
|
+
}));
|
|
2426
|
+
}
|
|
2427
|
+
catch (error) {
|
|
2428
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2429
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
2430
|
+
}
|
|
2431
|
+
return;
|
|
2432
|
+
}
|
|
2433
|
+
// Create a new metric YAML file in semantic-layer/metrics/
|
|
2434
|
+
if (req.method === 'POST' && path === '/api/semantic-layer/metric') {
|
|
2435
|
+
try {
|
|
2436
|
+
const body = await readJSON(req);
|
|
2437
|
+
const { name, label, description, domain, sql, type, table, tags } = body;
|
|
2438
|
+
if (!name || !sql || !type || !table) {
|
|
2439
|
+
res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2440
|
+
res.end(serializeJSON({ error: 'name, sql, type, and table are required' }));
|
|
2441
|
+
return;
|
|
2442
|
+
}
|
|
2443
|
+
const slug = name.toLowerCase().replace(/[^a-z0-9]+/g, '_').replace(/^_+|_+$/g, '');
|
|
2444
|
+
const metricsDir = join(projectRoot, 'semantic-layer', 'metrics');
|
|
2445
|
+
mkdirSync(metricsDir, { recursive: true });
|
|
2446
|
+
const filePath = join(metricsDir, `${slug}.yaml`);
|
|
2447
|
+
const tagList = Array.isArray(tags) && tags.length > 0
|
|
2448
|
+
? `\ntags:\n${tags.map(t => ` - ${t}`).join('\n')}`
|
|
2449
|
+
: '';
|
|
2450
|
+
const yaml = `name: ${slug}
|
|
2451
|
+
label: ${label || name}
|
|
2452
|
+
description: ${description || ''}
|
|
2453
|
+
domain: ${domain || 'general'}
|
|
2454
|
+
sql: ${sql}
|
|
2455
|
+
type: ${type}
|
|
2456
|
+
table: ${table}${tagList}
|
|
2457
|
+
`;
|
|
2458
|
+
writeFileSync(filePath, yaml, 'utf-8');
|
|
2459
|
+
res.writeHead(201, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2460
|
+
res.end(serializeJSON({ ok: true, path: `semantic-layer/metrics/${slug}.yaml` }));
|
|
2461
|
+
}
|
|
2462
|
+
catch (error) {
|
|
2463
|
+
res.writeHead(500, { 'Content-Type': 'application/json; charset=utf-8' });
|
|
2464
|
+
res.end(serializeJSON({ error: error instanceof Error ? error.message : String(error) }));
|
|
2465
|
+
}
|
|
2466
|
+
return;
|
|
2467
|
+
}
|
|
2468
|
+
if (req.method !== 'GET') {
|
|
2469
|
+
res.writeHead(405, { 'Content-Type': 'text/plain; charset=utf-8' });
|
|
2470
|
+
res.end('Method not allowed');
|
|
2471
|
+
return;
|
|
2472
|
+
}
|
|
2473
|
+
const requestedPath = path === '/' ? '/index.html' : path;
|
|
2474
|
+
const filePath = safeJoin(rootDir, requestedPath);
|
|
2475
|
+
if (!filePath || !existsSync(filePath) || statSync(filePath).isDirectory()) {
|
|
2476
|
+
res.writeHead(404, { 'Content-Type': 'text/html; charset=utf-8' });
|
|
2477
|
+
res.end(renderNotFound(path));
|
|
2478
|
+
return;
|
|
2479
|
+
}
|
|
2480
|
+
const content = readFileSync(filePath);
|
|
2481
|
+
res.writeHead(200, { 'Content-Type': contentTypeFor(filePath) });
|
|
2482
|
+
res.end(content);
|
|
2483
|
+
});
|
|
2484
|
+
return new Promise((resolvePromise, reject) => {
|
|
2485
|
+
let retriedWithRandomPort = false;
|
|
2486
|
+
server.on('error', (error) => {
|
|
2487
|
+
if (error.code === 'EADDRINUSE' && !retriedWithRandomPort) {
|
|
2488
|
+
retriedWithRandomPort = true;
|
|
2489
|
+
server.listen(0, bindHost);
|
|
2490
|
+
return;
|
|
2491
|
+
}
|
|
2492
|
+
reject(error);
|
|
2493
|
+
});
|
|
2494
|
+
server.listen(preferredPort, bindHost, () => {
|
|
2495
|
+
const address = server.address();
|
|
2496
|
+
if (!address || typeof address === 'string') {
|
|
2497
|
+
reject(new Error('Failed to resolve local server address.'));
|
|
2498
|
+
return;
|
|
2499
|
+
}
|
|
2500
|
+
resolvePromise(address.port);
|
|
2501
|
+
});
|
|
2502
|
+
});
|
|
2503
|
+
}
|
|
2504
|
+
export async function assertLocalQueryRuntimeReady(executor, connection) {
|
|
2505
|
+
try {
|
|
2506
|
+
const connector = await executor.getConnector(connection);
|
|
2507
|
+
const ok = await connector.ping();
|
|
2508
|
+
if (!ok) {
|
|
2509
|
+
throw new Error(`Connection check failed for driver "${connection.driver}".`);
|
|
2510
|
+
}
|
|
2511
|
+
}
|
|
2512
|
+
catch (error) {
|
|
2513
|
+
throw new Error(formatLocalQueryRuntimeError(connection, error));
|
|
2514
|
+
}
|
|
2515
|
+
}
|
|
2516
|
+
export function formatLocalQueryRuntimeError(connection, error) {
|
|
2517
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
2518
|
+
const driver = connection.driver;
|
|
2519
|
+
const currentNode = process.versions.node;
|
|
2520
|
+
if ((driver === 'file' || driver === 'duckdb') &&
|
|
2521
|
+
detail.includes('duckdb.node')) {
|
|
2522
|
+
return `Local query runtime is unavailable for driver "${driver}": DuckDB native bindings could not be loaded. Current Node.js runtime: ${currentNode}. Reinstall dependencies with a supported LTS Node release (for example Node 18, 20, or 22), then rerun "pnpm install". Original error: ${detail}`;
|
|
2523
|
+
}
|
|
2524
|
+
return `Local query runtime is unavailable for driver "${driver}": ${detail}`;
|
|
2525
|
+
}
|
|
2526
|
+
/**
|
|
2527
|
+
* Normalize connector QueryResult → SPA-friendly shape.
|
|
2528
|
+
* Connector returns columns as ColumnMeta[] ({name,type,driverType}).
|
|
2529
|
+
* The notebook SPA expects columns as string[] (just names).
|
|
2530
|
+
*/
|
|
2531
|
+
function normalizeQueryResult(result, semanticRefs) {
|
|
2532
|
+
const rawCols = Array.isArray(result?.columns) ? result.columns : [];
|
|
2533
|
+
const columns = rawCols.map((c) => typeof c === 'string' ? c : typeof c?.name === 'string' ? c.name : String(c));
|
|
2534
|
+
const hasRefs = semanticRefs && (semanticRefs.metrics.length > 0 || semanticRefs.dimensions.length > 0);
|
|
2535
|
+
return {
|
|
2536
|
+
columns,
|
|
2537
|
+
rows: Array.isArray(result?.rows) ? result.rows : [],
|
|
2538
|
+
rowCount: typeof result?.rowCount === 'number' ? result.rowCount : (result?.rows?.length ?? 0),
|
|
2539
|
+
executionTime: typeof result?.executionTimeMs === 'number'
|
|
2540
|
+
? result.executionTimeMs
|
|
2541
|
+
: typeof result?.executionTime === 'number'
|
|
2542
|
+
? result.executionTime
|
|
2543
|
+
: 0,
|
|
2544
|
+
...(hasRefs ? { semanticRefs } : {}),
|
|
2545
|
+
};
|
|
2546
|
+
}
|
|
2547
|
+
function isLLMProviderId(value) {
|
|
2548
|
+
return value === 'claude-agent-sdk'
|
|
2549
|
+
|| value === 'claude-code'
|
|
2550
|
+
|| value === 'openai'
|
|
2551
|
+
|| value === 'gemini'
|
|
2552
|
+
|| value === 'ollama';
|
|
2553
|
+
}
|
|
2554
|
+
function loadAppDashboard(projectRoot, appId, dashboardId) {
|
|
2555
|
+
for (const p of findAppDocuments(projectRoot)) {
|
|
2556
|
+
const { document: app } = loadAppDocument(p);
|
|
2557
|
+
if (!app || app.id !== appId)
|
|
2558
|
+
continue;
|
|
2559
|
+
const appDir = p.slice(0, -'/dql.app.json'.length);
|
|
2560
|
+
for (const d of findDashboardsForApp(appDir)) {
|
|
2561
|
+
const { document: dashboard } = loadDashboardDocument(d);
|
|
2562
|
+
if (dashboard?.id === dashboardId)
|
|
2563
|
+
return { app, dashboard };
|
|
2564
|
+
}
|
|
2565
|
+
}
|
|
2566
|
+
return null;
|
|
2567
|
+
}
|
|
2568
|
+
function resolveDashboardItemBlock(item, manifest) {
|
|
2569
|
+
if (isBlockIdRef(item.block)) {
|
|
2570
|
+
return manifest.blocks[item.block.blockId] ?? null;
|
|
2571
|
+
}
|
|
2572
|
+
const normalizedRef = normalize(item.block.ref).replaceAll('\\', '/');
|
|
2573
|
+
return Object.values(manifest.blocks).find((b) => normalize(b.filePath).replaceAll('\\', '/') === normalizedRef) ?? null;
|
|
2574
|
+
}
|
|
2575
|
+
export function serializeJSON(value) {
|
|
2576
|
+
return JSON.stringify(value, (_key, current) => {
|
|
2577
|
+
if (typeof current === 'bigint') {
|
|
2578
|
+
const asNumber = Number(current);
|
|
2579
|
+
return Number.isSafeInteger(asNumber) ? asNumber : current.toString();
|
|
2580
|
+
}
|
|
2581
|
+
return current;
|
|
2582
|
+
});
|
|
2583
|
+
}
|
|
2584
|
+
function renderNotFound(path) {
|
|
2585
|
+
return `<!doctype html>
|
|
2586
|
+
<html lang="en">
|
|
2587
|
+
<head>
|
|
2588
|
+
<meta charset="utf-8" />
|
|
2589
|
+
<title>DQL Local Runtime</title>
|
|
2590
|
+
<style>
|
|
2591
|
+
body { font-family: Inter, system-ui, sans-serif; margin: 40px; color: #111827; }
|
|
2592
|
+
code { background: #f3f4f6; padding: 2px 6px; border-radius: 6px; }
|
|
2593
|
+
</style>
|
|
2594
|
+
</head>
|
|
2595
|
+
<body>
|
|
2596
|
+
<h1>DQL Local Runtime</h1>
|
|
2597
|
+
<p>No file exists for <code>${escapeHtml(path)}</code>.</p>
|
|
2598
|
+
<p>Try opening <code>/</code> or confirm that you built the bundle correctly.</p>
|
|
2599
|
+
</body>
|
|
2600
|
+
</html>`;
|
|
2601
|
+
}
|
|
2602
|
+
function escapeHtml(value) {
|
|
2603
|
+
return value
|
|
2604
|
+
.replaceAll('&', '&')
|
|
2605
|
+
.replaceAll('<', '<')
|
|
2606
|
+
.replaceAll('>', '>')
|
|
2607
|
+
.replaceAll('"', '"')
|
|
2608
|
+
.replaceAll("'", ''');
|
|
2609
|
+
}
|
|
2610
|
+
export function findProjectRoot(startDir) {
|
|
2611
|
+
let current = resolve(startDir);
|
|
2612
|
+
while (true) {
|
|
2613
|
+
if (existsSync(join(current, 'dql.config.json'))) {
|
|
2614
|
+
return current;
|
|
2615
|
+
}
|
|
2616
|
+
const parent = dirname(current);
|
|
2617
|
+
if (parent === current) {
|
|
2618
|
+
return startDir;
|
|
2619
|
+
}
|
|
2620
|
+
current = parent;
|
|
2621
|
+
}
|
|
2622
|
+
}
|
|
2623
|
+
export function loadProjectConfig(projectRoot) {
|
|
2624
|
+
const configPath = join(projectRoot, 'dql.config.json');
|
|
2625
|
+
if (!existsSync(configPath)) {
|
|
2626
|
+
return {};
|
|
2627
|
+
}
|
|
2628
|
+
const raw = JSON.parse(readFileSync(configPath, 'utf-8'));
|
|
2629
|
+
const config = raw;
|
|
2630
|
+
// Normalize modern `connections.default` format to `defaultConnection`
|
|
2631
|
+
if (!config.defaultConnection && raw.connections) {
|
|
2632
|
+
const connections = raw.connections;
|
|
2633
|
+
const defaultConn = connections.default;
|
|
2634
|
+
if (defaultConn?.driver) {
|
|
2635
|
+
// Support both `filepath` (correct) and `path` (legacy/init compat)
|
|
2636
|
+
const filepath = (defaultConn.filepath ?? defaultConn.path);
|
|
2637
|
+
config.defaultConnection = {
|
|
2638
|
+
driver: defaultConn.driver,
|
|
2639
|
+
...(filepath ? { filepath } : {}),
|
|
2640
|
+
};
|
|
2641
|
+
}
|
|
2642
|
+
}
|
|
2643
|
+
return config;
|
|
2644
|
+
}
|
|
2645
|
+
export function prepareLocalExecution(sql, connection, projectRoot, projectConfig) {
|
|
2646
|
+
const normalizedConnection = normalizeProjectConnection(connection, projectRoot);
|
|
2647
|
+
return {
|
|
2648
|
+
sql: shouldResolveProjectPaths(normalizedConnection)
|
|
2649
|
+
? resolveProjectRelativeSqlPaths(sql, projectRoot, projectConfig.dataDir)
|
|
2650
|
+
: sql,
|
|
2651
|
+
connection: normalizedConnection,
|
|
2652
|
+
};
|
|
2653
|
+
}
|
|
2654
|
+
/**
|
|
2655
|
+
* Shared resolver for `@metric(name)` / `@dim(name)` refs in raw SQL.
|
|
2656
|
+
* Used by notebook SQL execution and Block Studio validation so both paths
|
|
2657
|
+
* behave identically. If the SQL has no refs, returns it unchanged.
|
|
2658
|
+
*/
|
|
2659
|
+
export function prepareSemanticSql(sql, semanticLayer) {
|
|
2660
|
+
if (!hasSemanticRefs(sql)) {
|
|
2661
|
+
return { sql, semanticRefs: { metrics: [], dimensions: [] }, unresolvedRefs: [] };
|
|
2662
|
+
}
|
|
2663
|
+
const resolution = resolveSemanticRefs(sql, semanticLayer);
|
|
2664
|
+
return {
|
|
2665
|
+
sql: resolution.resolvedSql,
|
|
2666
|
+
semanticRefs: {
|
|
2667
|
+
metrics: resolution.resolvedMetrics,
|
|
2668
|
+
dimensions: resolution.resolvedDimensions,
|
|
2669
|
+
},
|
|
2670
|
+
unresolvedRefs: resolution.unresolvedRefs,
|
|
2671
|
+
};
|
|
2672
|
+
}
|
|
2673
|
+
export function normalizeProjectConnection(connection, projectRoot) {
|
|
2674
|
+
const normalized = { ...connection };
|
|
2675
|
+
if ((normalized.driver === 'file' || normalized.driver === 'duckdb') && normalized.filepath && normalized.filepath !== ':memory:' && !isAbsoluteLikePath(normalized.filepath)) {
|
|
2676
|
+
normalized.filepath = resolve(projectRoot, normalized.filepath);
|
|
2677
|
+
}
|
|
2678
|
+
if (normalized.driver === 'sqlite' && normalized.database && normalized.database !== ':memory:' && !isAbsoluteLikePath(normalized.database)) {
|
|
2679
|
+
normalized.database = resolve(projectRoot, normalized.database);
|
|
2680
|
+
}
|
|
2681
|
+
return normalized;
|
|
2682
|
+
}
|
|
2683
|
+
export function resolveProjectRelativeSqlPaths(sql, projectRoot, dataDir) {
|
|
2684
|
+
const resolvedRoot = resolve(projectRoot);
|
|
2685
|
+
const normalizedDataDir = typeof dataDir === 'string' && dataDir.trim().length > 0
|
|
2686
|
+
? resolve(projectRoot, dataDir)
|
|
2687
|
+
: join(resolvedRoot, 'data');
|
|
2688
|
+
return sql.replace(/\b(read_csv_auto|read_csv|read_parquet|read_json_auto|read_json|read_ndjson_auto|read_ndjson|read_xlsx|parquet_scan)\s*\(\s*(['"])(\.{1,2}\/[^'"]*)\2/gi, (_match, fnName, quote, relativePath) => {
|
|
2689
|
+
const absolutePath = relativePath.startsWith('./data/')
|
|
2690
|
+
? join(normalizedDataDir, relativePath.slice('./data/'.length))
|
|
2691
|
+
: resolve(resolvedRoot, relativePath);
|
|
2692
|
+
return `${fnName}(${quote}${absolutePath.replaceAll('\\', '/')}${quote}`;
|
|
2693
|
+
});
|
|
2694
|
+
}
|
|
2695
|
+
function shouldResolveProjectPaths(connection) {
|
|
2696
|
+
return connection.driver === 'file' || connection.driver === 'duckdb' || connection.driver === 'sqlite';
|
|
2697
|
+
}
|
|
2698
|
+
function isAbsoluteLikePath(value) {
|
|
2699
|
+
return value.startsWith('/') || value.startsWith('\\') || /^[A-Za-z]:[\\/]/.test(value);
|
|
2700
|
+
}
|
|
2701
|
+
function readJSON(req) {
|
|
2702
|
+
return new Promise((resolvePromise, reject) => {
|
|
2703
|
+
const chunks = [];
|
|
2704
|
+
req.on('data', (chunk) => chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)));
|
|
2705
|
+
req.on('end', () => {
|
|
2706
|
+
try {
|
|
2707
|
+
const raw = Buffer.concat(chunks).toString('utf-8');
|
|
2708
|
+
resolvePromise(raw ? JSON.parse(raw) : {});
|
|
2709
|
+
}
|
|
2710
|
+
catch (error) {
|
|
2711
|
+
reject(error);
|
|
2712
|
+
}
|
|
2713
|
+
});
|
|
2714
|
+
req.on('error', reject);
|
|
2715
|
+
});
|
|
2716
|
+
}
|
|
2717
|
+
function safeJoin(rootDir, requestPath) {
|
|
2718
|
+
const normalized = normalize(requestPath).replace(/^([.][.][/\\])+/, '');
|
|
2719
|
+
const fullPath = resolve(rootDir, `.${normalized.startsWith('/') ? normalized : `/${normalized}`}`);
|
|
2720
|
+
const resolvedRoot = resolve(rootDir);
|
|
2721
|
+
return fullPath.startsWith(resolvedRoot) ? fullPath : null;
|
|
2722
|
+
}
|
|
2723
|
+
function contentTypeFor(filePath) {
|
|
2724
|
+
switch (extname(filePath)) {
|
|
2725
|
+
case '.html':
|
|
2726
|
+
return 'text/html; charset=utf-8';
|
|
2727
|
+
case '.json':
|
|
2728
|
+
return 'application/json; charset=utf-8';
|
|
2729
|
+
case '.js':
|
|
2730
|
+
return 'application/javascript; charset=utf-8';
|
|
2731
|
+
case '.css':
|
|
2732
|
+
return 'text/css; charset=utf-8';
|
|
2733
|
+
case '.svg':
|
|
2734
|
+
return 'image/svg+xml';
|
|
2735
|
+
case '.woff2':
|
|
2736
|
+
return 'font/woff2';
|
|
2737
|
+
case '.woff':
|
|
2738
|
+
return 'font/woff';
|
|
2739
|
+
default:
|
|
2740
|
+
return 'text/plain; charset=utf-8';
|
|
2741
|
+
}
|
|
2742
|
+
}
|
|
2743
|
+
function listProjectFiles(projectRoot) {
|
|
2744
|
+
const allowed = new Set(['.dql', '.sql', '.md', '.json', '.csv', '.yaml', '.yml', '.dqlnb']);
|
|
2745
|
+
const files = [];
|
|
2746
|
+
walk(projectRoot);
|
|
2747
|
+
return files.sort();
|
|
2748
|
+
function walk(currentDir) {
|
|
2749
|
+
for (const entry of readdirSync(currentDir)) {
|
|
2750
|
+
if (entry === 'node_modules' || entry === '.git' || entry === 'dist') {
|
|
2751
|
+
continue;
|
|
2752
|
+
}
|
|
2753
|
+
const fullPath = join(currentDir, entry);
|
|
2754
|
+
const stat = statSync(fullPath);
|
|
2755
|
+
if (stat.isDirectory()) {
|
|
2756
|
+
walk(fullPath);
|
|
2757
|
+
continue;
|
|
2758
|
+
}
|
|
2759
|
+
if (allowed.has(extname(entry))) {
|
|
2760
|
+
files.push(fullPath.slice(projectRoot.length + 1));
|
|
2761
|
+
}
|
|
2762
|
+
}
|
|
2763
|
+
}
|
|
2764
|
+
}
|
|
2765
|
+
function resolveNotebook(projectRoot, projectTitle) {
|
|
2766
|
+
const notebookPath = join(projectRoot, 'notebooks', 'welcome.dqlnb');
|
|
2767
|
+
if (existsSync(notebookPath)) {
|
|
2768
|
+
return deserializeNotebook(readFileSync(notebookPath, 'utf-8'));
|
|
2769
|
+
}
|
|
2770
|
+
return createWelcomeNotebook('starter', projectTitle);
|
|
2771
|
+
}
|
|
2772
|
+
function normalizeNotebookCell(value) {
|
|
2773
|
+
if (!value || typeof value !== 'object') {
|
|
2774
|
+
return null;
|
|
2775
|
+
}
|
|
2776
|
+
const candidate = value;
|
|
2777
|
+
if (typeof candidate.id !== 'string' || typeof candidate.type !== 'string' || typeof candidate.source !== 'string') {
|
|
2778
|
+
return null;
|
|
2779
|
+
}
|
|
2780
|
+
return {
|
|
2781
|
+
id: candidate.id,
|
|
2782
|
+
type: candidate.type,
|
|
2783
|
+
source: candidate.source,
|
|
2784
|
+
title: typeof candidate.title === 'string' ? candidate.title : undefined,
|
|
2785
|
+
config: candidate.config,
|
|
2786
|
+
};
|
|
2787
|
+
}
|
|
2788
|
+
function isConnectionConfig(value) {
|
|
2789
|
+
return Boolean(value && typeof value === 'object' && 'driver' in value);
|
|
2790
|
+
}
|
|
2791
|
+
function scanNotebookFiles(projectRoot) {
|
|
2792
|
+
const result = [];
|
|
2793
|
+
const folderMap = {
|
|
2794
|
+
notebooks: 'notebook',
|
|
2795
|
+
workbooks: 'workbook',
|
|
2796
|
+
blocks: 'block',
|
|
2797
|
+
dashboards: 'dashboard',
|
|
2798
|
+
};
|
|
2799
|
+
for (const [folder, type] of Object.entries(folderMap)) {
|
|
2800
|
+
const dir = join(projectRoot, folder);
|
|
2801
|
+
if (!existsSync(dir))
|
|
2802
|
+
continue;
|
|
2803
|
+
collect(dir, folder, type);
|
|
2804
|
+
}
|
|
2805
|
+
return result;
|
|
2806
|
+
function collect(currentDir, relativeDir, type) {
|
|
2807
|
+
try {
|
|
2808
|
+
for (const entry of readdirSync(currentDir, { withFileTypes: true })) {
|
|
2809
|
+
const fullPath = join(currentDir, entry.name);
|
|
2810
|
+
const relativePath = `${relativeDir}/${entry.name}`;
|
|
2811
|
+
if (entry.isDirectory()) {
|
|
2812
|
+
collect(fullPath, relativePath, type);
|
|
2813
|
+
continue;
|
|
2814
|
+
}
|
|
2815
|
+
if (!entry.isFile())
|
|
2816
|
+
continue;
|
|
2817
|
+
if (!entry.name.endsWith('.dql') && !entry.name.endsWith('.dqlnb'))
|
|
2818
|
+
continue;
|
|
2819
|
+
result.push({
|
|
2820
|
+
name: entry.name.replace(/\.(dql|dqlnb)$/, ''),
|
|
2821
|
+
path: relativePath,
|
|
2822
|
+
type,
|
|
2823
|
+
folder: relativeDir.split('/')[0] ?? relativeDir,
|
|
2824
|
+
});
|
|
2825
|
+
}
|
|
2826
|
+
}
|
|
2827
|
+
catch { /* skip unreadable dirs */ }
|
|
2828
|
+
}
|
|
2829
|
+
}
|
|
2830
|
+
function scanDataFiles(projectRoot) {
|
|
2831
|
+
const dataDir = join(projectRoot, 'data');
|
|
2832
|
+
if (!existsSync(dataDir))
|
|
2833
|
+
return [];
|
|
2834
|
+
try {
|
|
2835
|
+
return readdirSync(dataDir, { withFileTypes: true })
|
|
2836
|
+
.filter((e) => e.isFile() && /\.(csv|parquet|json)$/.test(e.name))
|
|
2837
|
+
.map((e) => ({ name: e.name, path: `data/${e.name}`, columns: [] }));
|
|
2838
|
+
}
|
|
2839
|
+
catch {
|
|
2840
|
+
return [];
|
|
2841
|
+
}
|
|
2842
|
+
}
|
|
2843
|
+
function readUserPrefs(userPrefsPath) {
|
|
2844
|
+
try {
|
|
2845
|
+
if (!existsSync(userPrefsPath)) {
|
|
2846
|
+
return { favorites: [], recentlyUsed: [] };
|
|
2847
|
+
}
|
|
2848
|
+
const raw = JSON.parse(readFileSync(userPrefsPath, 'utf-8'));
|
|
2849
|
+
return {
|
|
2850
|
+
favorites: Array.isArray(raw.favorites) ? raw.favorites.map(String) : [],
|
|
2851
|
+
recentlyUsed: Array.isArray(raw.recentlyUsed) ? raw.recentlyUsed.map(String) : [],
|
|
2852
|
+
};
|
|
2853
|
+
}
|
|
2854
|
+
catch {
|
|
2855
|
+
return { favorites: [], recentlyUsed: [] };
|
|
2856
|
+
}
|
|
2857
|
+
}
|
|
2858
|
+
function writeUserPrefs(userPrefsPath, prefs) {
|
|
2859
|
+
writeFileSync(userPrefsPath, JSON.stringify(prefs, null, 2) + '\n', 'utf-8');
|
|
2860
|
+
}
|
|
2861
|
+
async function introspectSchema(executor, connection) {
|
|
2862
|
+
let tables = [];
|
|
2863
|
+
let columnsByPath = new Map();
|
|
2864
|
+
// Tier 1: information_schema (PG, MySQL, Snowflake, MSSQL, DuckDB, Redshift, Fabric, Databricks)
|
|
2865
|
+
try {
|
|
2866
|
+
const catalogRows = await executor.executeQuery(`SELECT table_schema, table_name, table_type
|
|
2867
|
+
FROM information_schema.tables
|
|
2868
|
+
WHERE table_schema NOT IN ('information_schema', 'pg_catalog')
|
|
2869
|
+
ORDER BY table_schema, table_name`, [], {}, connection);
|
|
2870
|
+
tables = catalogRows.rows.map((row) => {
|
|
2871
|
+
const schema = String(row['table_schema'] ?? row['TABLE_SCHEMA'] ?? 'default');
|
|
2872
|
+
const name = String(row['table_name'] ?? row['TABLE_NAME'] ?? '');
|
|
2873
|
+
const type = String(row['table_type'] ?? row['TABLE_TYPE'] ?? 'TABLE');
|
|
2874
|
+
const path = schema ? `${schema}.${name}` : name;
|
|
2875
|
+
return { schema, name, path, type };
|
|
2876
|
+
});
|
|
2877
|
+
const columnRows = await executor.executeQuery(`SELECT table_schema, table_name, column_name, data_type
|
|
2878
|
+
FROM information_schema.columns
|
|
2879
|
+
WHERE table_schema NOT IN ('information_schema', 'pg_catalog')
|
|
2880
|
+
ORDER BY table_schema, table_name, ordinal_position`, [], {}, connection);
|
|
2881
|
+
columnsByPath = columnRows.rows.reduce((map, row) => {
|
|
2882
|
+
const schema = String(row['table_schema'] ?? row['TABLE_SCHEMA'] ?? 'default');
|
|
2883
|
+
const tableName = String(row['table_name'] ?? row['TABLE_NAME'] ?? '');
|
|
2884
|
+
const path = schema ? `${schema}.${tableName}` : tableName;
|
|
2885
|
+
const next = map.get(path) ?? [];
|
|
2886
|
+
next.push({
|
|
2887
|
+
name: String(row['column_name'] ?? row['COLUMN_NAME'] ?? ''),
|
|
2888
|
+
type: String(row['data_type'] ?? row['DATA_TYPE'] ?? ''),
|
|
2889
|
+
});
|
|
2890
|
+
map.set(path, next);
|
|
2891
|
+
return map;
|
|
2892
|
+
}, new Map());
|
|
2893
|
+
return { tables, columnsByPath };
|
|
2894
|
+
}
|
|
2895
|
+
catch {
|
|
2896
|
+
// Tier 1 failed — try connector methods
|
|
2897
|
+
}
|
|
2898
|
+
// Tier 2: connector.listTables() + connector.listColumns() (SQLite, BigQuery, Athena, ClickHouse, Trino)
|
|
2899
|
+
try {
|
|
2900
|
+
const connector = await executor.getConnector(connection);
|
|
2901
|
+
if (typeof connector.listTables === 'function') {
|
|
2902
|
+
const rawTables = await connector.listTables();
|
|
2903
|
+
tables = rawTables.map((t) => {
|
|
2904
|
+
const schema = t.schema || 'default';
|
|
2905
|
+
const path = t.schema ? `${t.schema}.${t.name}` : t.name;
|
|
2906
|
+
return { schema, name: t.name, path, type: t.type };
|
|
2907
|
+
});
|
|
2908
|
+
}
|
|
2909
|
+
if (typeof connector.listColumns === 'function') {
|
|
2910
|
+
const rawColumns = await connector.listColumns();
|
|
2911
|
+
columnsByPath = rawColumns.reduce((map, col) => {
|
|
2912
|
+
const schema = col.schema || 'default';
|
|
2913
|
+
const path = schema ? `${schema}.${col.table}` : col.table;
|
|
2914
|
+
const next = map.get(path) ?? [];
|
|
2915
|
+
next.push({ name: col.name, type: col.dataType });
|
|
2916
|
+
map.set(path, next);
|
|
2917
|
+
return map;
|
|
2918
|
+
}, new Map());
|
|
2919
|
+
}
|
|
2920
|
+
}
|
|
2921
|
+
catch {
|
|
2922
|
+
// Tier 3: tables only, no columns — already have what we have
|
|
2923
|
+
}
|
|
2924
|
+
return { tables, columnsByPath };
|
|
2925
|
+
}
|
|
2926
|
+
function buildDatabaseSchemaTree(projectRoot, executor, connection) {
|
|
2927
|
+
return (async () => {
|
|
2928
|
+
const dataFiles = scanDataFiles(projectRoot);
|
|
2929
|
+
const { tables: dbTables, columnsByPath: dbColumnsByPath } = await introspectSchema(executor, connection);
|
|
2930
|
+
const schemaMap = new Map();
|
|
2931
|
+
for (const table of dbTables) {
|
|
2932
|
+
const schemaName = table.schema || 'default';
|
|
2933
|
+
const existing = schemaMap.get(schemaName) ?? [];
|
|
2934
|
+
existing.push({ name: table.name, path: table.path, type: table.type });
|
|
2935
|
+
schemaMap.set(schemaName, existing);
|
|
2936
|
+
}
|
|
2937
|
+
const databaseNodes = Array.from(schemaMap.entries())
|
|
2938
|
+
.sort((a, b) => a[0].localeCompare(b[0]))
|
|
2939
|
+
.map(([schemaName, tables]) => ({
|
|
2940
|
+
id: `db-schema:${schemaName}`,
|
|
2941
|
+
label: schemaName,
|
|
2942
|
+
kind: 'schema',
|
|
2943
|
+
children: tables
|
|
2944
|
+
.sort((a, b) => a.name.localeCompare(b.name))
|
|
2945
|
+
.map((table) => ({
|
|
2946
|
+
id: `db-table:${table.path}`,
|
|
2947
|
+
label: table.name,
|
|
2948
|
+
kind: 'table',
|
|
2949
|
+
path: table.path,
|
|
2950
|
+
type: table.type,
|
|
2951
|
+
children: (dbColumnsByPath.get(table.path) ?? []).map((column) => ({
|
|
2952
|
+
id: `db-column:${table.path}:${column.name}`,
|
|
2953
|
+
label: column.name,
|
|
2954
|
+
kind: 'column',
|
|
2955
|
+
path: table.path,
|
|
2956
|
+
type: column.type,
|
|
2957
|
+
})),
|
|
2958
|
+
})),
|
|
2959
|
+
}));
|
|
2960
|
+
// Eagerly resolve file columns via DuckDB DESCRIBE
|
|
2961
|
+
if (dataFiles.length > 0) {
|
|
2962
|
+
const fileChildren = [];
|
|
2963
|
+
for (const file of dataFiles) {
|
|
2964
|
+
let columns = [];
|
|
2965
|
+
try {
|
|
2966
|
+
const ext = file.name.split('.').pop()?.toLowerCase();
|
|
2967
|
+
const readFn = ext === 'parquet' ? 'read_parquet' : ext === 'json' ? 'read_json_auto' : 'read_csv_auto';
|
|
2968
|
+
const descResult = await executor.executeQuery(`DESCRIBE SELECT * FROM ${readFn}('${file.path.replace(/'/g, "''")}') LIMIT 0`, [], {}, connection);
|
|
2969
|
+
columns = descResult.rows.map((row) => ({
|
|
2970
|
+
id: `db-column:${file.path}:${String(row['column_name'] ?? '')}`,
|
|
2971
|
+
label: String(row['column_name'] ?? ''),
|
|
2972
|
+
kind: 'column',
|
|
2973
|
+
path: file.path,
|
|
2974
|
+
type: String(row['column_type'] ?? ''),
|
|
2975
|
+
}));
|
|
2976
|
+
}
|
|
2977
|
+
catch {
|
|
2978
|
+
// file column discovery failed — empty children is fine
|
|
2979
|
+
}
|
|
2980
|
+
fileChildren.push({
|
|
2981
|
+
id: `db-table:${file.path}`,
|
|
2982
|
+
label: file.name,
|
|
2983
|
+
kind: 'table',
|
|
2984
|
+
path: file.path,
|
|
2985
|
+
type: 'FILE',
|
|
2986
|
+
children: columns,
|
|
2987
|
+
});
|
|
2988
|
+
}
|
|
2989
|
+
databaseNodes.unshift({
|
|
2990
|
+
id: 'db-schema:files',
|
|
2991
|
+
label: 'files',
|
|
2992
|
+
kind: 'schema',
|
|
2993
|
+
children: fileChildren,
|
|
2994
|
+
});
|
|
2995
|
+
}
|
|
2996
|
+
return databaseNodes;
|
|
2997
|
+
})();
|
|
2998
|
+
}
|
|
2999
|
+
function openBlockStudioDocument(projectRoot, relativePath, semanticLayer) {
|
|
3000
|
+
const normalizedPath = normalize(relativePath).replace(/^\/+/, '');
|
|
3001
|
+
if (!normalizedPath.startsWith('blocks/')) {
|
|
3002
|
+
throw new Error('Invalid block path');
|
|
3003
|
+
}
|
|
3004
|
+
const absPath = join(projectRoot, normalizedPath);
|
|
3005
|
+
if (!existsSync(absPath)) {
|
|
3006
|
+
throw new Error(`File not found: ${normalizedPath}`);
|
|
3007
|
+
}
|
|
3008
|
+
const source = readFileSync(absPath, 'utf-8');
|
|
3009
|
+
const companionPath = blockCompanionRelativePath(normalizedPath);
|
|
3010
|
+
const companion = companionPath ? readBlockCompanionFile(projectRoot, companionPath) : null;
|
|
3011
|
+
const parsedMetadata = parseBlockSourceMetadata(source);
|
|
3012
|
+
const fileName = normalizedPath.split('/').pop()?.replace(/\.dql$/, '') ?? 'block';
|
|
3013
|
+
const metadata = {
|
|
3014
|
+
name: parsedMetadata.name || companion?.name || fileName,
|
|
3015
|
+
path: normalizedPath,
|
|
3016
|
+
domain: parsedMetadata.domain || companion?.domain || normalizedPath.split('/').slice(1, -1).join('/') || 'uncategorized',
|
|
3017
|
+
description: parsedMetadata.description || companion?.description || '',
|
|
3018
|
+
owner: parsedMetadata.owner || companion?.owner || '',
|
|
3019
|
+
tags: parsedMetadata.tags.length > 0 ? parsedMetadata.tags : companion?.tags ?? [],
|
|
3020
|
+
reviewStatus: companion?.reviewStatus,
|
|
3021
|
+
};
|
|
3022
|
+
return {
|
|
3023
|
+
path: normalizedPath,
|
|
3024
|
+
source,
|
|
3025
|
+
metadata,
|
|
3026
|
+
companionPath: companionPath && existsSync(join(projectRoot, companionPath)) ? companionPath : null,
|
|
3027
|
+
validation: validateBlockStudioSource(source, semanticLayer),
|
|
3028
|
+
};
|
|
3029
|
+
}
|
|
3030
|
+
function parseBlockStudioArrayField(source, key) {
|
|
3031
|
+
const match = source.match(new RegExp(`\\b${key}\\s*=\\s*\\[([\\s\\S]*?)\\]`, 'i'));
|
|
3032
|
+
if (!match)
|
|
3033
|
+
return [];
|
|
3034
|
+
return (match[1].match(/"([^"]*)"/g) ?? []).map((value) => value.slice(1, -1)).filter(Boolean);
|
|
3035
|
+
}
|
|
3036
|
+
function parseBlockStudioStringField(source, key) {
|
|
3037
|
+
return source.match(new RegExp(`\\b${key}\\s*=\\s*"([^"]*)"`, 'i'))?.[1] ?? undefined;
|
|
3038
|
+
}
|
|
3039
|
+
function parseSemanticBlockConfig(source) {
|
|
3040
|
+
const blockType = (parseBlockStudioStringField(source, 'type') ?? 'custom').toLowerCase() === 'semantic'
|
|
3041
|
+
? 'semantic'
|
|
3042
|
+
: 'custom';
|
|
3043
|
+
const metric = parseBlockStudioStringField(source, 'metric');
|
|
3044
|
+
const metrics = parseBlockStudioArrayField(source, 'metrics');
|
|
3045
|
+
const dimensions = parseBlockStudioArrayField(source, 'dimensions');
|
|
3046
|
+
const timeDimension = parseBlockStudioStringField(source, 'time_dimension');
|
|
3047
|
+
const granularity = parseBlockStudioStringField(source, 'granularity');
|
|
3048
|
+
const limitMatch = source.match(/\blimit\s*=\s*(\d+)/i);
|
|
3049
|
+
return {
|
|
3050
|
+
blockType,
|
|
3051
|
+
metric,
|
|
3052
|
+
metrics,
|
|
3053
|
+
dimensions,
|
|
3054
|
+
timeDimension,
|
|
3055
|
+
granularity,
|
|
3056
|
+
limit: limitMatch ? Number.parseInt(limitMatch[1], 10) : undefined,
|
|
3057
|
+
};
|
|
3058
|
+
}
|
|
3059
|
+
function buildSemanticTableMapping(semanticLayer, rows) {
|
|
3060
|
+
const dbTableNames = new Set();
|
|
3061
|
+
const schemaQualified = new Map();
|
|
3062
|
+
for (const row of rows) {
|
|
3063
|
+
const schema = String(row['table_schema'] ?? '');
|
|
3064
|
+
const name = String(row['table_name'] ?? '');
|
|
3065
|
+
if (!name)
|
|
3066
|
+
continue;
|
|
3067
|
+
dbTableNames.add(name);
|
|
3068
|
+
schemaQualified.set(name, schema ? `${schema}.${name}` : name);
|
|
3069
|
+
}
|
|
3070
|
+
const tableMapping = {};
|
|
3071
|
+
const allSemanticTables = new Set();
|
|
3072
|
+
for (const metric of semanticLayer.listMetrics())
|
|
3073
|
+
allSemanticTables.add(metric.table);
|
|
3074
|
+
for (const dimension of semanticLayer.listDimensions())
|
|
3075
|
+
allSemanticTables.add(dimension.table);
|
|
3076
|
+
for (const semTable of allSemanticTables) {
|
|
3077
|
+
if (dbTableNames.has(semTable) && schemaQualified.has(semTable)) {
|
|
3078
|
+
tableMapping[semTable] = schemaQualified.get(semTable);
|
|
3079
|
+
}
|
|
3080
|
+
}
|
|
3081
|
+
return Object.keys(tableMapping).length > 0 ? tableMapping : undefined;
|
|
3082
|
+
}
|
|
3083
|
+
function isDbtSemanticRuntime(projectConfig, detectedProvider, semanticLayer) {
|
|
3084
|
+
if (projectConfig.semanticLayer?.provider === 'dbt' || detectedProvider === 'dbt')
|
|
3085
|
+
return true;
|
|
3086
|
+
return Boolean(semanticLayer?.listMetrics().some((metric) => metric.source?.provider === 'dbt'));
|
|
3087
|
+
}
|
|
3088
|
+
function composeRuntimeSemanticQuery(request, semanticLayer, context) {
|
|
3089
|
+
const useMetricFlow = request.engine === 'metricflow' || (request.engine !== 'native' &&
|
|
3090
|
+
isDbtSemanticRuntime(context.projectConfig, context.detectedProvider, semanticLayer));
|
|
3091
|
+
if (useMetricFlow) {
|
|
3092
|
+
const dbtProjectPath = context.projectConfig.semanticLayer?.projectPath;
|
|
3093
|
+
const compiled = compileMetricFlowQuery({
|
|
3094
|
+
projectRoot: context.projectRoot,
|
|
3095
|
+
dbtProjectPath,
|
|
3096
|
+
metrics: request.metrics,
|
|
3097
|
+
dimensions: request.dimensions,
|
|
3098
|
+
filters: request.filters,
|
|
3099
|
+
timeDimension: request.timeDimension,
|
|
3100
|
+
orderBy: request.orderBy,
|
|
3101
|
+
limit: request.limit,
|
|
3102
|
+
savedQuery: request.savedQuery,
|
|
3103
|
+
});
|
|
3104
|
+
return {
|
|
3105
|
+
sql: compiled.sql,
|
|
3106
|
+
joins: [],
|
|
3107
|
+
tables: [],
|
|
3108
|
+
engine: 'metricflow',
|
|
3109
|
+
};
|
|
3110
|
+
}
|
|
3111
|
+
const composed = semanticLayer.composeQuery({
|
|
3112
|
+
metrics: request.metrics,
|
|
3113
|
+
dimensions: request.dimensions,
|
|
3114
|
+
filters: request.filters,
|
|
3115
|
+
limit: request.limit,
|
|
3116
|
+
timeDimension: request.timeDimension,
|
|
3117
|
+
orderBy: request.orderBy,
|
|
3118
|
+
driver: context.driver,
|
|
3119
|
+
tableMapping: context.tableMapping,
|
|
3120
|
+
});
|
|
3121
|
+
return composed ? { ...composed, engine: 'native' } : null;
|
|
3122
|
+
}
|
|
3123
|
+
function composeSemanticBlockSql(source, semanticLayer, options) {
|
|
3124
|
+
const config = parseSemanticBlockConfig(source);
|
|
3125
|
+
const metrics = config.metrics.length > 0
|
|
3126
|
+
? config.metrics
|
|
3127
|
+
: config.metric
|
|
3128
|
+
? [config.metric]
|
|
3129
|
+
: [];
|
|
3130
|
+
const semanticRefs = {
|
|
3131
|
+
metrics,
|
|
3132
|
+
dimensions: config.dimensions,
|
|
3133
|
+
segments: [],
|
|
3134
|
+
};
|
|
3135
|
+
const diagnostics = [];
|
|
3136
|
+
if (config.blockType !== 'semantic') {
|
|
3137
|
+
return { sql: null, diagnostics, semanticRefs };
|
|
3138
|
+
}
|
|
3139
|
+
if (metrics.length === 0) {
|
|
3140
|
+
diagnostics.push({
|
|
3141
|
+
severity: 'error',
|
|
3142
|
+
code: 'semantic_metric_missing',
|
|
3143
|
+
message: 'Semantic block is missing a metric. Add metric = "metric_name" or metrics = ["metric_name"].',
|
|
3144
|
+
});
|
|
3145
|
+
return { sql: null, diagnostics, semanticRefs };
|
|
3146
|
+
}
|
|
3147
|
+
if (config.timeDimension && !config.granularity) {
|
|
3148
|
+
diagnostics.push({
|
|
3149
|
+
severity: 'error',
|
|
3150
|
+
code: 'semantic_granularity_missing',
|
|
3151
|
+
message: `Semantic block selects time_dimension = "${config.timeDimension}" but is missing granularity.`,
|
|
3152
|
+
});
|
|
3153
|
+
}
|
|
3154
|
+
const refValidation = semanticLayer.validateReferences([...metrics, ...config.dimensions]);
|
|
3155
|
+
for (const unknown of refValidation.unknown) {
|
|
3156
|
+
diagnostics.push({
|
|
3157
|
+
severity: 'error',
|
|
3158
|
+
code: 'semantic_ref',
|
|
3159
|
+
message: `Unknown semantic reference: ${unknown}`,
|
|
3160
|
+
});
|
|
3161
|
+
}
|
|
3162
|
+
if (diagnostics.some((diagnostic) => diagnostic.severity === 'error')) {
|
|
3163
|
+
return { sql: null, diagnostics, semanticRefs };
|
|
3164
|
+
}
|
|
3165
|
+
let composed;
|
|
3166
|
+
try {
|
|
3167
|
+
composed = options?.projectRoot && options.projectConfig
|
|
3168
|
+
? composeRuntimeSemanticQuery({
|
|
3169
|
+
metrics,
|
|
3170
|
+
dimensions: config.dimensions,
|
|
3171
|
+
timeDimension: config.timeDimension && config.granularity
|
|
3172
|
+
? { name: config.timeDimension, granularity: config.granularity }
|
|
3173
|
+
: undefined,
|
|
3174
|
+
limit: config.limit,
|
|
3175
|
+
}, semanticLayer, {
|
|
3176
|
+
projectRoot: options.projectRoot,
|
|
3177
|
+
projectConfig: options.projectConfig,
|
|
3178
|
+
detectedProvider: options.detectedProvider ?? null,
|
|
3179
|
+
driver: options.driver,
|
|
3180
|
+
tableMapping: options.tableMapping,
|
|
3181
|
+
})
|
|
3182
|
+
: semanticLayer.composeQuery({
|
|
3183
|
+
metrics,
|
|
3184
|
+
dimensions: config.dimensions,
|
|
3185
|
+
timeDimension: config.timeDimension && config.granularity
|
|
3186
|
+
? { name: config.timeDimension, granularity: config.granularity }
|
|
3187
|
+
: undefined,
|
|
3188
|
+
limit: config.limit,
|
|
3189
|
+
driver: options?.driver,
|
|
3190
|
+
tableMapping: options?.tableMapping,
|
|
3191
|
+
});
|
|
3192
|
+
}
|
|
3193
|
+
catch (error) {
|
|
3194
|
+
diagnostics.push({
|
|
3195
|
+
severity: 'error',
|
|
3196
|
+
code: error instanceof MetricFlowUnavailableError ? 'metricflow_unavailable' : 'semantic_compose_failed',
|
|
3197
|
+
message: error instanceof Error ? error.message : String(error),
|
|
3198
|
+
});
|
|
3199
|
+
return { sql: null, diagnostics, semanticRefs };
|
|
3200
|
+
}
|
|
3201
|
+
if (!composed) {
|
|
3202
|
+
diagnostics.push({
|
|
3203
|
+
severity: 'error',
|
|
3204
|
+
code: 'semantic_compose_failed',
|
|
3205
|
+
message: `Could not compose SQL for semantic block metrics: [${metrics.join(', ')}].`,
|
|
3206
|
+
});
|
|
3207
|
+
return { sql: null, diagnostics, semanticRefs };
|
|
3208
|
+
}
|
|
3209
|
+
return {
|
|
3210
|
+
sql: composed.sql,
|
|
3211
|
+
diagnostics,
|
|
3212
|
+
semanticRefs,
|
|
3213
|
+
};
|
|
3214
|
+
}
|
|
3215
|
+
function resolveCustomBlockSql(sql, semanticLayer) {
|
|
3216
|
+
if (!sql) {
|
|
3217
|
+
return {
|
|
3218
|
+
sql: null,
|
|
3219
|
+
diagnostics: [],
|
|
3220
|
+
semanticRefs: { metrics: [], dimensions: [], segments: [] },
|
|
3221
|
+
};
|
|
3222
|
+
}
|
|
3223
|
+
const semanticRefs = extractBlockStudioSemanticReferences(sql);
|
|
3224
|
+
if (!hasSemanticRefs(sql)) {
|
|
3225
|
+
return { sql, diagnostics: [], semanticRefs };
|
|
3226
|
+
}
|
|
3227
|
+
const resolution = resolveSemanticRefs(sql, semanticLayer);
|
|
3228
|
+
if (resolution.unresolvedRefs.length > 0) {
|
|
3229
|
+
return {
|
|
3230
|
+
sql: null,
|
|
3231
|
+
diagnostics: resolution.unresolvedRefs.map((unresolved) => ({
|
|
3232
|
+
severity: 'error',
|
|
3233
|
+
code: 'semantic_ref',
|
|
3234
|
+
message: `Unknown semantic reference: ${unresolved}`,
|
|
3235
|
+
})),
|
|
3236
|
+
semanticRefs,
|
|
3237
|
+
};
|
|
3238
|
+
}
|
|
3239
|
+
return {
|
|
3240
|
+
sql: resolution.resolvedSql,
|
|
3241
|
+
diagnostics: [],
|
|
3242
|
+
semanticRefs: {
|
|
3243
|
+
metrics: resolution.resolvedMetrics,
|
|
3244
|
+
dimensions: resolution.resolvedDimensions,
|
|
3245
|
+
segments: semanticRefs.segments,
|
|
3246
|
+
},
|
|
3247
|
+
};
|
|
3248
|
+
}
|
|
3249
|
+
export function validateBlockStudioSource(source, semanticLayer) {
|
|
3250
|
+
const diagnostics = [];
|
|
3251
|
+
const semanticConfig = parseSemanticBlockConfig(source);
|
|
3252
|
+
if (semanticConfig.blockType !== 'semantic') {
|
|
3253
|
+
try {
|
|
3254
|
+
const parser = new Parser(source, '<block-studio>');
|
|
3255
|
+
parser.parse();
|
|
3256
|
+
}
|
|
3257
|
+
catch (error) {
|
|
3258
|
+
diagnostics.push({
|
|
3259
|
+
severity: 'error',
|
|
3260
|
+
code: 'syntax',
|
|
3261
|
+
message: error instanceof Error ? error.message : String(error),
|
|
3262
|
+
});
|
|
3263
|
+
}
|
|
3264
|
+
}
|
|
3265
|
+
else {
|
|
3266
|
+
const hasBlockHeader = /\bblock\s+"[^"]+"\s*\{/i.test(source);
|
|
3267
|
+
const hasClosingBrace = /\}\s*$/m.test(source);
|
|
3268
|
+
if (!hasBlockHeader || !hasClosingBrace) {
|
|
3269
|
+
diagnostics.push({
|
|
3270
|
+
severity: 'error',
|
|
3271
|
+
code: 'semantic_shape',
|
|
3272
|
+
message: 'Semantic block must use block "Name" { ... } structure.',
|
|
3273
|
+
});
|
|
3274
|
+
}
|
|
3275
|
+
}
|
|
3276
|
+
let semanticRefs = extractBlockStudioSemanticReferences(source);
|
|
3277
|
+
if (semanticConfig.blockType === 'semantic') {
|
|
3278
|
+
const selectedMetrics = semanticConfig.metrics.length > 0
|
|
3279
|
+
? semanticConfig.metrics
|
|
3280
|
+
: semanticConfig.metric
|
|
3281
|
+
? [semanticConfig.metric]
|
|
3282
|
+
: [];
|
|
3283
|
+
semanticRefs = {
|
|
3284
|
+
metrics: selectedMetrics,
|
|
3285
|
+
dimensions: semanticConfig.dimensions,
|
|
3286
|
+
segments: semanticRefs.segments,
|
|
3287
|
+
};
|
|
3288
|
+
}
|
|
3289
|
+
let executableSql = extractBlockStudioSql(source);
|
|
3290
|
+
if (semanticConfig.blockType === 'semantic') {
|
|
3291
|
+
if (semanticLayer) {
|
|
3292
|
+
const semanticCompose = composeSemanticBlockSql(source, semanticLayer);
|
|
3293
|
+
semanticRefs = semanticCompose.semanticRefs;
|
|
3294
|
+
diagnostics.push(...semanticCompose.diagnostics);
|
|
3295
|
+
executableSql = semanticCompose.sql;
|
|
3296
|
+
}
|
|
3297
|
+
else {
|
|
3298
|
+
diagnostics.push({
|
|
3299
|
+
severity: 'error',
|
|
3300
|
+
code: 'semantic_layer_missing',
|
|
3301
|
+
message: 'Semantic block cannot run because no semantic layer is configured.',
|
|
3302
|
+
});
|
|
3303
|
+
executableSql = null;
|
|
3304
|
+
}
|
|
3305
|
+
}
|
|
3306
|
+
else if (semanticLayer) {
|
|
3307
|
+
const resolvedCustomSql = resolveCustomBlockSql(executableSql, semanticLayer);
|
|
3308
|
+
semanticRefs = resolvedCustomSql.semanticRefs;
|
|
3309
|
+
diagnostics.push(...resolvedCustomSql.diagnostics);
|
|
3310
|
+
executableSql = resolvedCustomSql.sql;
|
|
3311
|
+
}
|
|
3312
|
+
const chartConfig = extractBlockStudioChartConfig(source);
|
|
3313
|
+
if (!chartConfig) {
|
|
3314
|
+
diagnostics.push({
|
|
3315
|
+
severity: 'warning',
|
|
3316
|
+
code: 'visualization_missing',
|
|
3317
|
+
message: 'Block has no visualization section yet.',
|
|
3318
|
+
});
|
|
3319
|
+
}
|
|
3320
|
+
if (!executableSql) {
|
|
3321
|
+
diagnostics.push(semanticConfig.blockType === 'semantic'
|
|
3322
|
+
? {
|
|
3323
|
+
severity: 'warning',
|
|
3324
|
+
code: 'semantic_not_runnable',
|
|
3325
|
+
message: 'Semantic block is not runnable yet. Select a metric and complete any required time settings.',
|
|
3326
|
+
}
|
|
3327
|
+
: {
|
|
3328
|
+
severity: 'warning',
|
|
3329
|
+
code: 'sql_missing',
|
|
3330
|
+
message: 'No executable SQL found in the block source.',
|
|
3331
|
+
});
|
|
3332
|
+
}
|
|
3333
|
+
return {
|
|
3334
|
+
valid: diagnostics.every((diagnostic) => diagnostic.severity !== 'error'),
|
|
3335
|
+
diagnostics,
|
|
3336
|
+
semanticRefs,
|
|
3337
|
+
chartConfig: chartConfig ?? undefined,
|
|
3338
|
+
executableSql,
|
|
3339
|
+
};
|
|
3340
|
+
}
|
|
3341
|
+
function saveBlockStudioArtifacts(projectRoot, options) {
|
|
3342
|
+
const slug = options.name.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '') || 'block';
|
|
3343
|
+
const safeDomain = (options.domain ?? '')
|
|
3344
|
+
.trim()
|
|
3345
|
+
.toLowerCase()
|
|
3346
|
+
.replace(/[^a-z0-9/_-]+/g, '-')
|
|
3347
|
+
.replace(/^\/+|\/+$/g, '') || 'uncategorized';
|
|
3348
|
+
const targetRelativePath = `blocks/${safeDomain}/${slug}.dql`;
|
|
3349
|
+
const targetPath = join(projectRoot, targetRelativePath);
|
|
3350
|
+
const previousPath = options.currentPath ? normalize(options.currentPath).replace(/^\/+/, '') : null;
|
|
3351
|
+
if (existsSync(targetPath) && previousPath !== targetRelativePath) {
|
|
3352
|
+
throw new Error('BLOCK_EXISTS');
|
|
3353
|
+
}
|
|
3354
|
+
mkdirSync(dirname(targetPath), { recursive: true });
|
|
3355
|
+
writeFileSync(targetPath, options.source.trimEnd() + '\n', 'utf-8');
|
|
3356
|
+
writeBlockCompanionFile(projectRoot, {
|
|
3357
|
+
slug,
|
|
3358
|
+
name: options.name,
|
|
3359
|
+
domain: safeDomain,
|
|
3360
|
+
description: options.description,
|
|
3361
|
+
owner: options.owner,
|
|
3362
|
+
tags: options.tags,
|
|
3363
|
+
provider: 'dql',
|
|
3364
|
+
content: options.source,
|
|
3365
|
+
});
|
|
3366
|
+
if (previousPath && previousPath !== targetRelativePath) {
|
|
3367
|
+
const previousAbsPath = join(projectRoot, previousPath);
|
|
3368
|
+
if (existsSync(previousAbsPath))
|
|
3369
|
+
rmSync(previousAbsPath, { force: true });
|
|
3370
|
+
const previousCompanion = blockCompanionRelativePath(previousPath);
|
|
3371
|
+
if (previousCompanion) {
|
|
3372
|
+
const previousCompanionPath = join(projectRoot, previousCompanion);
|
|
3373
|
+
if (existsSync(previousCompanionPath))
|
|
3374
|
+
rmSync(previousCompanionPath, { force: true });
|
|
3375
|
+
}
|
|
3376
|
+
}
|
|
3377
|
+
return targetRelativePath;
|
|
3378
|
+
}
|
|
3379
|
+
function blockCompanionRelativePath(blockPath) {
|
|
3380
|
+
const normalized = normalize(blockPath).replace(/^\/+/, '');
|
|
3381
|
+
if (!normalized.startsWith('blocks/'))
|
|
3382
|
+
return null;
|
|
3383
|
+
const withoutRoot = normalized.slice('blocks/'.length).replace(/\.dql$/, '.yaml');
|
|
3384
|
+
return join('semantic-layer', 'blocks', withoutRoot).replaceAll('\\', '/');
|
|
3385
|
+
}
|
|
3386
|
+
function readBlockCompanionFile(projectRoot, relativePath) {
|
|
3387
|
+
const absPath = join(projectRoot, relativePath);
|
|
3388
|
+
if (!existsSync(absPath))
|
|
3389
|
+
return null;
|
|
3390
|
+
try {
|
|
3391
|
+
const content = readFileSync(absPath, 'utf-8');
|
|
3392
|
+
const lines = content.split(/\r?\n/);
|
|
3393
|
+
const topLevel = {};
|
|
3394
|
+
const arrays = {};
|
|
3395
|
+
let currentArray = null;
|
|
3396
|
+
for (const rawLine of lines) {
|
|
3397
|
+
const line = rawLine.replace(/\t/g, ' ');
|
|
3398
|
+
if (!line.trim() || line.trimStart().startsWith('#'))
|
|
3399
|
+
continue;
|
|
3400
|
+
if (/^\S[^:]*:\s*$/.test(line)) {
|
|
3401
|
+
currentArray = line.trim().slice(0, -1);
|
|
3402
|
+
if (['tags', 'lineage', 'semanticMetrics', 'semanticDimensions'].includes(currentArray)) {
|
|
3403
|
+
arrays[currentArray] = [];
|
|
3404
|
+
}
|
|
3405
|
+
continue;
|
|
3406
|
+
}
|
|
3407
|
+
const itemMatch = line.match(/^\s*-\s*(.+)\s*$/);
|
|
3408
|
+
if (itemMatch && currentArray && arrays[currentArray]) {
|
|
3409
|
+
arrays[currentArray].push(parseYamlScalar(itemMatch[1]));
|
|
3410
|
+
continue;
|
|
3411
|
+
}
|
|
3412
|
+
const scalarMatch = line.match(/^([A-Za-z0-9_]+):\s*(.+)\s*$/);
|
|
3413
|
+
if (scalarMatch) {
|
|
3414
|
+
currentArray = null;
|
|
3415
|
+
topLevel[scalarMatch[1]] = parseYamlScalar(scalarMatch[2]);
|
|
3416
|
+
}
|
|
3417
|
+
}
|
|
3418
|
+
return {
|
|
3419
|
+
name: topLevel.name ?? '',
|
|
3420
|
+
block: topLevel.block ?? '',
|
|
3421
|
+
domain: topLevel.domain ?? '',
|
|
3422
|
+
description: topLevel.description ?? '',
|
|
3423
|
+
owner: topLevel.owner ?? '',
|
|
3424
|
+
tags: arrays.tags ?? [],
|
|
3425
|
+
reviewStatus: topLevel.reviewStatus ?? '',
|
|
3426
|
+
};
|
|
3427
|
+
}
|
|
3428
|
+
catch {
|
|
3429
|
+
return null;
|
|
3430
|
+
}
|
|
3431
|
+
}
|
|
3432
|
+
function parseBlockSourceMetadata(source) {
|
|
3433
|
+
const name = source.match(/^\s*block\s+"([^"]+)"/i)?.[1] ?? '';
|
|
3434
|
+
const extractString = (key) => source.match(new RegExp(`\\b${key}\\s*=\\s*"([^"]*)"`, 'i'))?.[1] ?? '';
|
|
3435
|
+
const tags = source.match(/\btags\s*=\s*\[([^\]]*)\]/i);
|
|
3436
|
+
return {
|
|
3437
|
+
name,
|
|
3438
|
+
domain: extractString('domain'),
|
|
3439
|
+
description: extractString('description'),
|
|
3440
|
+
owner: extractString('owner'),
|
|
3441
|
+
tags: tags ? (tags[1].match(/"([^"]*)"/g) ?? []).map((value) => value.slice(1, -1)) : [],
|
|
3442
|
+
};
|
|
3443
|
+
}
|
|
3444
|
+
function extractBlockStudioChartConfig(source) {
|
|
3445
|
+
const vizMatch = source.match(/visualization\s*\{([^}]+)\}/is);
|
|
3446
|
+
if (!vizMatch)
|
|
3447
|
+
return null;
|
|
3448
|
+
const body = vizMatch[1];
|
|
3449
|
+
const get = (key) => body.match(new RegExp(`\\b${key}\\s*=\\s*["']?([\\w-]+)["']?`, 'i'))?.[1];
|
|
3450
|
+
const chart = get('chart');
|
|
3451
|
+
if (!chart)
|
|
3452
|
+
return null;
|
|
3453
|
+
const title = body.match(/\btitle\s*=\s*"([^"]+)"/i)?.[1];
|
|
3454
|
+
return {
|
|
3455
|
+
chart,
|
|
3456
|
+
x: get('x'),
|
|
3457
|
+
y: get('y'),
|
|
3458
|
+
color: get('color'),
|
|
3459
|
+
title,
|
|
3460
|
+
};
|
|
3461
|
+
}
|
|
3462
|
+
function extractBlockStudioSql(source) {
|
|
3463
|
+
const tripleQuoteMatch = source.match(/query\s*=\s*"""([\s\S]*?)"""/i);
|
|
3464
|
+
if (tripleQuoteMatch)
|
|
3465
|
+
return tripleQuoteMatch[1].trim() || null;
|
|
3466
|
+
const bareTripleMatch = source.match(/"""([\s\S]*?)"""/);
|
|
3467
|
+
if (bareTripleMatch)
|
|
3468
|
+
return bareTripleMatch[1].trim() || null;
|
|
3469
|
+
if (/^\s*(dashboard|workbook)\s+"/i.test(source))
|
|
3470
|
+
return null;
|
|
3471
|
+
const sqlKeywordMatch = source.match(/\b(SELECT|WITH|INSERT|UPDATE|DELETE|CREATE|DROP|ALTER|SHOW|DESCRIBE|EXPLAIN)\b([\s\S]*)/i);
|
|
3472
|
+
if (!sqlKeywordMatch)
|
|
3473
|
+
return null;
|
|
3474
|
+
let raw = sqlKeywordMatch[0];
|
|
3475
|
+
const dqlSectionStart = raw.search(/\b(visualization|tests|block)\s*\{/i);
|
|
3476
|
+
if (dqlSectionStart > 0)
|
|
3477
|
+
raw = raw.slice(0, dqlSectionStart);
|
|
3478
|
+
return raw.trim() || null;
|
|
3479
|
+
}
|
|
3480
|
+
function extractBlockStudioSemanticReferences(source) {
|
|
3481
|
+
const metrics = new Set();
|
|
3482
|
+
const dimensions = new Set();
|
|
3483
|
+
const segments = new Set();
|
|
3484
|
+
const semanticRegex = /@(metric|dim)\(([^)]+)\)/gi;
|
|
3485
|
+
let match;
|
|
3486
|
+
while ((match = semanticRegex.exec(source))) {
|
|
3487
|
+
const name = match[2].trim();
|
|
3488
|
+
if (!name)
|
|
3489
|
+
continue;
|
|
3490
|
+
if (match[1].toLowerCase() === 'metric')
|
|
3491
|
+
metrics.add(name);
|
|
3492
|
+
else
|
|
3493
|
+
dimensions.add(name);
|
|
3494
|
+
}
|
|
3495
|
+
const segmentRegex = /\/\*\s*segment:([^*]+)\*\//gi;
|
|
3496
|
+
while ((match = segmentRegex.exec(source))) {
|
|
3497
|
+
const name = match[1].trim();
|
|
3498
|
+
if (name)
|
|
3499
|
+
segments.add(name);
|
|
3500
|
+
}
|
|
3501
|
+
return {
|
|
3502
|
+
metrics: Array.from(metrics),
|
|
3503
|
+
dimensions: Array.from(dimensions),
|
|
3504
|
+
segments: Array.from(segments),
|
|
3505
|
+
};
|
|
3506
|
+
}
|
|
3507
|
+
function canonicalizeSafe(source) {
|
|
3508
|
+
try {
|
|
3509
|
+
return canonicalize(source);
|
|
3510
|
+
}
|
|
3511
|
+
catch {
|
|
3512
|
+
// If the block body has content the parser rejects (e.g. unsupported
|
|
3513
|
+
// syntax in a user-provided template), keep the original bytes rather
|
|
3514
|
+
// than fail the write — format header gets added next time it passes fmt.
|
|
3515
|
+
return source;
|
|
3516
|
+
}
|
|
3517
|
+
}
|
|
3518
|
+
function canonicalizeNotebookSafe(source) {
|
|
3519
|
+
try {
|
|
3520
|
+
return canonicalizeNotebook(source);
|
|
3521
|
+
}
|
|
3522
|
+
catch {
|
|
3523
|
+
return source;
|
|
3524
|
+
}
|
|
3525
|
+
}
|
|
3526
|
+
export function readGitMetadata(projectRoot) {
|
|
3527
|
+
const run = (cmd) => execSync(cmd, { cwd: projectRoot, encoding: 'utf-8', timeout: 5000, stdio: ['ignore', 'pipe', 'ignore'] }).toString().trim();
|
|
3528
|
+
try {
|
|
3529
|
+
const commitSha = run('git rev-parse HEAD');
|
|
3530
|
+
let repo = null;
|
|
3531
|
+
let branch = null;
|
|
3532
|
+
try {
|
|
3533
|
+
repo = run('git config --get remote.origin.url') || null;
|
|
3534
|
+
}
|
|
3535
|
+
catch { /* no remote */ }
|
|
3536
|
+
try {
|
|
3537
|
+
branch = run('git rev-parse --abbrev-ref HEAD') || null;
|
|
3538
|
+
}
|
|
3539
|
+
catch { /* detached */ }
|
|
3540
|
+
return { commitSha, repo, branch };
|
|
3541
|
+
}
|
|
3542
|
+
catch {
|
|
3543
|
+
return null;
|
|
3544
|
+
}
|
|
3545
|
+
}
|
|
3546
|
+
export function createBlockArtifacts(projectRoot, options) {
|
|
3547
|
+
const slug = options.name.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '') || 'block';
|
|
3548
|
+
const safeDomain = (options.domain ?? '')
|
|
3549
|
+
.trim()
|
|
3550
|
+
.toLowerCase()
|
|
3551
|
+
.replace(/[^a-z0-9/_-]+/g, '-')
|
|
3552
|
+
.replace(/^\/+|\/+$/g, '');
|
|
3553
|
+
const blocksDir = safeDomain ? join(projectRoot, 'blocks', safeDomain) : join(projectRoot, 'blocks');
|
|
3554
|
+
mkdirSync(blocksDir, { recursive: true });
|
|
3555
|
+
const blockPath = join(blocksDir, `${slug}.dql`);
|
|
3556
|
+
if (existsSync(blockPath)) {
|
|
3557
|
+
throw new Error('BLOCK_EXISTS');
|
|
3558
|
+
}
|
|
3559
|
+
const templateContent = options.template
|
|
3560
|
+
? listBlockTemplates().find((template) => template.id === options.template)?.content
|
|
3561
|
+
: undefined;
|
|
3562
|
+
const relativePath = safeDomain ? `blocks/${safeDomain}/${slug}.dql` : `blocks/${slug}.dql`;
|
|
3563
|
+
const fileContent = canonicalizeSafe(normalizeBlockStudioContent({
|
|
3564
|
+
name: options.name,
|
|
3565
|
+
domain: safeDomain || 'uncategorized',
|
|
3566
|
+
owner: options.owner,
|
|
3567
|
+
description: options.description,
|
|
3568
|
+
tags: options.tags,
|
|
3569
|
+
llmContext: options.llmContext,
|
|
3570
|
+
examples: options.examples,
|
|
3571
|
+
invariants: options.invariants,
|
|
3572
|
+
content: options.content?.trim() || templateContent,
|
|
3573
|
+
}));
|
|
3574
|
+
writeFileSync(blockPath, fileContent, 'utf-8');
|
|
3575
|
+
const companionPath = writeBlockCompanionFile(projectRoot, {
|
|
3576
|
+
slug,
|
|
3577
|
+
name: options.name,
|
|
3578
|
+
domain: safeDomain || 'uncategorized',
|
|
3579
|
+
owner: options.owner,
|
|
3580
|
+
description: options.description,
|
|
3581
|
+
tags: options.tags,
|
|
3582
|
+
provider: 'dql',
|
|
3583
|
+
content: fileContent,
|
|
3584
|
+
gitMetadata: options.gitMetadata,
|
|
3585
|
+
gitPath: relativePath,
|
|
3586
|
+
});
|
|
3587
|
+
return {
|
|
3588
|
+
path: relativePath,
|
|
3589
|
+
content: fileContent,
|
|
3590
|
+
companionPath,
|
|
3591
|
+
};
|
|
3592
|
+
}
|
|
3593
|
+
export function createSemanticBuilderBlock(projectRoot, options) {
|
|
3594
|
+
const slug = options.name.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '') || 'block';
|
|
3595
|
+
const safeDomain = (options.domain ?? '')
|
|
3596
|
+
.trim()
|
|
3597
|
+
.toLowerCase()
|
|
3598
|
+
.replace(/[^a-z0-9/_-]+/g, '-')
|
|
3599
|
+
.replace(/^\/+|\/+$/g, '') || 'uncategorized';
|
|
3600
|
+
const blocksDir = join(projectRoot, 'blocks', safeDomain);
|
|
3601
|
+
mkdirSync(blocksDir, { recursive: true });
|
|
3602
|
+
const blockPath = join(blocksDir, `${slug}.dql`);
|
|
3603
|
+
if (existsSync(blockPath)) {
|
|
3604
|
+
throw new Error('BLOCK_EXISTS');
|
|
3605
|
+
}
|
|
3606
|
+
const content = canonicalizeSafe(options.blockType === 'custom'
|
|
3607
|
+
? buildCustomSemanticBlockContent(options)
|
|
3608
|
+
: buildSemanticBlockContent(options));
|
|
3609
|
+
writeFileSync(blockPath, content, 'utf-8');
|
|
3610
|
+
const companionPath = writeBlockCompanionFile(projectRoot, {
|
|
3611
|
+
slug,
|
|
3612
|
+
name: options.name,
|
|
3613
|
+
domain: safeDomain,
|
|
3614
|
+
description: options.description,
|
|
3615
|
+
owner: options.owner,
|
|
3616
|
+
tags: options.tags,
|
|
3617
|
+
provider: options.provider,
|
|
3618
|
+
content,
|
|
3619
|
+
lineage: options.tables,
|
|
3620
|
+
semanticMetrics: options.metrics,
|
|
3621
|
+
semanticDimensions: [
|
|
3622
|
+
...options.dimensions,
|
|
3623
|
+
...(options.timeDimension ? [options.timeDimension.name] : []),
|
|
3624
|
+
],
|
|
3625
|
+
});
|
|
3626
|
+
return {
|
|
3627
|
+
path: `blocks/${safeDomain}/${slug}.dql`,
|
|
3628
|
+
content,
|
|
3629
|
+
companionPath,
|
|
3630
|
+
};
|
|
3631
|
+
}
|
|
3632
|
+
function buildSemanticBlockContent(options) {
|
|
3633
|
+
const lines = [
|
|
3634
|
+
`block "${options.name}" {`,
|
|
3635
|
+
` domain = "${options.domain ?? 'uncategorized'}"`,
|
|
3636
|
+
' type = "semantic"',
|
|
3637
|
+
];
|
|
3638
|
+
if (options.description)
|
|
3639
|
+
lines.push(` description = "${escapeDqlString(options.description)}"`);
|
|
3640
|
+
if (options.owner)
|
|
3641
|
+
lines.push(` owner = "${escapeDqlString(options.owner)}"`);
|
|
3642
|
+
if (options.tags && options.tags.length > 0) {
|
|
3643
|
+
lines.push(` tags = [${options.tags.map((tag) => `"${escapeDqlString(tag)}"`).join(', ')}]`);
|
|
3644
|
+
}
|
|
3645
|
+
if (options.metrics.length === 1) {
|
|
3646
|
+
lines.push(` metric = "${escapeDqlString(options.metrics[0])}"`);
|
|
3647
|
+
}
|
|
3648
|
+
else {
|
|
3649
|
+
lines.push(` metrics = [${options.metrics.map((metric) => `"${escapeDqlString(metric)}"`).join(', ')}]`);
|
|
3650
|
+
}
|
|
3651
|
+
if (options.dimensions.length > 0) {
|
|
3652
|
+
lines.push(` dimensions = [${options.dimensions.map((dimension) => `"${escapeDqlString(dimension)}"`).join(', ')}]`);
|
|
3653
|
+
}
|
|
3654
|
+
if (options.timeDimension) {
|
|
3655
|
+
lines.push(` time_dimension = "${escapeDqlString(options.timeDimension.name)}"`);
|
|
3656
|
+
lines.push(` granularity = "${escapeDqlString(options.timeDimension.granularity)}"`);
|
|
3657
|
+
}
|
|
3658
|
+
const visualization = buildVisualizationBlock(options.chart ?? 'table', options.dimensions, options.timeDimension, options.metrics);
|
|
3659
|
+
if (visualization) {
|
|
3660
|
+
lines.push('');
|
|
3661
|
+
lines.push(...visualization);
|
|
3662
|
+
}
|
|
3663
|
+
lines.push('}');
|
|
3664
|
+
return lines.join('\n') + '\n';
|
|
3665
|
+
}
|
|
3666
|
+
function buildCustomSemanticBlockContent(options) {
|
|
3667
|
+
const lines = [
|
|
3668
|
+
`block "${options.name}" {`,
|
|
3669
|
+
` domain = "${options.domain ?? 'uncategorized'}"`,
|
|
3670
|
+
' type = "custom"',
|
|
3671
|
+
];
|
|
3672
|
+
if (options.description)
|
|
3673
|
+
lines.push(` description = "${escapeDqlString(options.description)}"`);
|
|
3674
|
+
if (options.owner)
|
|
3675
|
+
lines.push(` owner = "${escapeDqlString(options.owner)}"`);
|
|
3676
|
+
if (options.tags && options.tags.length > 0) {
|
|
3677
|
+
lines.push(` tags = [${options.tags.map((tag) => `"${escapeDqlString(tag)}"`).join(', ')}]`);
|
|
3678
|
+
}
|
|
3679
|
+
lines.push('');
|
|
3680
|
+
lines.push(' query = """');
|
|
3681
|
+
lines.push(...indentBlock(options.sql.trim(), 8).split('\n'));
|
|
3682
|
+
lines.push(' """');
|
|
3683
|
+
const visualization = buildVisualizationBlock(options.chart ?? 'table', options.dimensions, options.timeDimension, options.metrics);
|
|
3684
|
+
if (visualization) {
|
|
3685
|
+
lines.push('');
|
|
3686
|
+
lines.push(...visualization);
|
|
3687
|
+
}
|
|
3688
|
+
lines.push('}');
|
|
3689
|
+
return lines.join('\n') + '\n';
|
|
3690
|
+
}
|
|
3691
|
+
function buildVisualizationBlock(chart, dimensions, timeDimension, metrics) {
|
|
3692
|
+
const x = timeDimension ? `${timeDimension.name}_${timeDimension.granularity}` : dimensions[0];
|
|
3693
|
+
const y = metrics[0];
|
|
3694
|
+
if (!x && chart !== 'kpi' && chart !== 'table')
|
|
3695
|
+
return null;
|
|
3696
|
+
if (chart === 'table') {
|
|
3697
|
+
return [' visualization {', ' chart = "table"', ' }'];
|
|
3698
|
+
}
|
|
3699
|
+
if (chart === 'kpi') {
|
|
3700
|
+
return [' visualization {', ' chart = "kpi"', ` y = ${y}`, ' }'];
|
|
3701
|
+
}
|
|
3702
|
+
return [
|
|
3703
|
+
' visualization {',
|
|
3704
|
+
` chart = "${chart}"`,
|
|
3705
|
+
` x = ${x}`,
|
|
3706
|
+
` y = ${y}`,
|
|
3707
|
+
' }',
|
|
3708
|
+
];
|
|
3709
|
+
}
|
|
3710
|
+
function writeBlockCompanionFile(projectRoot, options) {
|
|
3711
|
+
const extractedRefs = extractSemanticReferenceNames(options.content);
|
|
3712
|
+
const semanticMetrics = Array.from(new Set([...(options.semanticMetrics ?? []), ...extractedRefs.metrics]));
|
|
3713
|
+
const semanticDimensions = Array.from(new Set([...(options.semanticDimensions ?? []), ...extractedRefs.dimensions]));
|
|
3714
|
+
const companionDir = join(projectRoot, 'semantic-layer', 'blocks', options.domain);
|
|
3715
|
+
mkdirSync(companionDir, { recursive: true });
|
|
3716
|
+
const companionPath = join(companionDir, `${options.slug}.yaml`);
|
|
3717
|
+
const lines = [
|
|
3718
|
+
`name: ${options.slug}`,
|
|
3719
|
+
`block: ${options.slug}`,
|
|
3720
|
+
`domain: ${options.domain}`,
|
|
3721
|
+
`description: ${yamlScalar(options.description?.trim() || options.name)}`,
|
|
3722
|
+
];
|
|
3723
|
+
if (options.owner)
|
|
3724
|
+
lines.push(`owner: ${yamlScalar(options.owner)}`);
|
|
3725
|
+
if (options.tags && options.tags.length > 0) {
|
|
3726
|
+
lines.push('tags:');
|
|
3727
|
+
for (const tag of options.tags)
|
|
3728
|
+
lines.push(` - ${yamlScalar(tag)}`);
|
|
3729
|
+
}
|
|
3730
|
+
if (options.provider) {
|
|
3731
|
+
lines.push('source:');
|
|
3732
|
+
lines.push(` provider: ${yamlScalar(options.provider)}`);
|
|
3733
|
+
lines.push(' objectType: block');
|
|
3734
|
+
lines.push(` objectId: ${yamlScalar(options.slug)}`);
|
|
3735
|
+
}
|
|
3736
|
+
if (semanticMetrics.length > 0) {
|
|
3737
|
+
lines.push('semanticMetrics:');
|
|
3738
|
+
for (const metric of semanticMetrics)
|
|
3739
|
+
lines.push(` - ${yamlScalar(metric)}`);
|
|
3740
|
+
}
|
|
3741
|
+
if (semanticDimensions.length > 0) {
|
|
3742
|
+
lines.push('semanticDimensions:');
|
|
3743
|
+
for (const dimension of semanticDimensions)
|
|
3744
|
+
lines.push(` - ${yamlScalar(dimension)}`);
|
|
3745
|
+
}
|
|
3746
|
+
const mappingEntries = [
|
|
3747
|
+
...semanticMetrics.map((metric) => [metric, metric]),
|
|
3748
|
+
...semanticDimensions.map((dimension) => [dimension, dimension]),
|
|
3749
|
+
];
|
|
3750
|
+
if (mappingEntries.length > 0) {
|
|
3751
|
+
lines.push('semanticMappings:');
|
|
3752
|
+
for (const [key, value] of mappingEntries) {
|
|
3753
|
+
lines.push(` ${key}: ${yamlScalar(value)}`);
|
|
3754
|
+
}
|
|
3755
|
+
}
|
|
3756
|
+
if (options.lineage && options.lineage.length > 0) {
|
|
3757
|
+
lines.push('lineage:');
|
|
3758
|
+
for (const table of options.lineage)
|
|
3759
|
+
lines.push(` - ${yamlScalar(table)}`);
|
|
3760
|
+
}
|
|
3761
|
+
if (options.gitMetadata || options.gitPath) {
|
|
3762
|
+
lines.push('git:');
|
|
3763
|
+
if (options.gitMetadata?.commitSha)
|
|
3764
|
+
lines.push(` commitSha: ${yamlScalar(options.gitMetadata.commitSha)}`);
|
|
3765
|
+
if (options.gitMetadata?.repo)
|
|
3766
|
+
lines.push(` repo: ${yamlScalar(options.gitMetadata.repo)}`);
|
|
3767
|
+
if (options.gitMetadata?.branch)
|
|
3768
|
+
lines.push(` branch: ${yamlScalar(options.gitMetadata.branch)}`);
|
|
3769
|
+
if (options.gitPath)
|
|
3770
|
+
lines.push(` path: ${yamlScalar(options.gitPath)}`);
|
|
3771
|
+
}
|
|
3772
|
+
lines.push('reviewStatus: draft');
|
|
3773
|
+
writeFileSync(companionPath, lines.join('\n') + '\n', 'utf-8');
|
|
3774
|
+
return relative(projectRoot, companionPath).replaceAll('\\', '/');
|
|
3775
|
+
}
|
|
3776
|
+
function extractSemanticReferenceNames(content) {
|
|
3777
|
+
const metrics = new Set();
|
|
3778
|
+
const dimensions = new Set();
|
|
3779
|
+
const regex = /@(metric|dim)\(([^)]+)\)/gi;
|
|
3780
|
+
let match;
|
|
3781
|
+
while ((match = regex.exec(content))) {
|
|
3782
|
+
const name = match[2].trim();
|
|
3783
|
+
if (!name)
|
|
3784
|
+
continue;
|
|
3785
|
+
if (match[1].toLowerCase() === 'metric') {
|
|
3786
|
+
metrics.add(name);
|
|
3787
|
+
}
|
|
3788
|
+
else {
|
|
3789
|
+
dimensions.add(name);
|
|
3790
|
+
}
|
|
3791
|
+
}
|
|
3792
|
+
return {
|
|
3793
|
+
metrics: Array.from(metrics),
|
|
3794
|
+
dimensions: Array.from(dimensions),
|
|
3795
|
+
};
|
|
3796
|
+
}
|
|
3797
|
+
function escapeDqlString(value) {
|
|
3798
|
+
return value.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
|
|
3799
|
+
}
|
|
3800
|
+
function indentBlock(value, spaces) {
|
|
3801
|
+
const prefix = ' '.repeat(spaces);
|
|
3802
|
+
return value.split('\n').map((line) => `${prefix}${line}`).join('\n');
|
|
3803
|
+
}
|
|
3804
|
+
function normalizeBlockStudioContent(options) {
|
|
3805
|
+
const content = options.content?.trim();
|
|
3806
|
+
if (content && /^\s*block\s+"/i.test(content)) {
|
|
3807
|
+
return `${content.trimEnd()}\n`;
|
|
3808
|
+
}
|
|
3809
|
+
return buildBlankBlockContent({
|
|
3810
|
+
name: options.name,
|
|
3811
|
+
domain: options.domain,
|
|
3812
|
+
owner: options.owner,
|
|
3813
|
+
description: options.description,
|
|
3814
|
+
tags: options.tags,
|
|
3815
|
+
llmContext: options.llmContext,
|
|
3816
|
+
examples: options.examples,
|
|
3817
|
+
invariants: options.invariants,
|
|
3818
|
+
sql: content || 'SELECT 1 AS value',
|
|
3819
|
+
});
|
|
3820
|
+
}
|
|
3821
|
+
function buildBlankBlockContent(options) {
|
|
3822
|
+
const lines = [
|
|
3823
|
+
`block "${escapeDqlString(options.name)}" {`,
|
|
3824
|
+
` domain = "${escapeDqlString(options.domain)}"`,
|
|
3825
|
+
' type = "custom"',
|
|
3826
|
+
` description = "${escapeDqlString(options.description?.trim() || options.name)}"`,
|
|
3827
|
+
` owner = "${escapeDqlString(options.owner?.trim() ?? '')}"`,
|
|
3828
|
+
];
|
|
3829
|
+
lines.push(` tags = [${(options.tags ?? []).map((tag) => `"${escapeDqlString(tag)}"`).join(', ')}]`);
|
|
3830
|
+
if (options.llmContext && options.llmContext.trim()) {
|
|
3831
|
+
lines.push(` llmContext = "${escapeDqlString(options.llmContext.trim())}"`);
|
|
3832
|
+
}
|
|
3833
|
+
if (options.invariants && options.invariants.length > 0) {
|
|
3834
|
+
lines.push(` invariants = [${options.invariants
|
|
3835
|
+
.filter((inv) => inv && inv.trim())
|
|
3836
|
+
.map((inv) => `"${escapeDqlString(inv.trim())}"`)
|
|
3837
|
+
.join(', ')}]`);
|
|
3838
|
+
}
|
|
3839
|
+
if (options.examples && options.examples.length > 0) {
|
|
3840
|
+
const items = options.examples.filter((ex) => ex.question && ex.question.trim());
|
|
3841
|
+
if (items.length > 0) {
|
|
3842
|
+
lines.push(' examples = [');
|
|
3843
|
+
for (const ex of items) {
|
|
3844
|
+
const parts = [`question = "${escapeDqlString(ex.question.trim())}"`];
|
|
3845
|
+
if (ex.sql && ex.sql.trim())
|
|
3846
|
+
parts.push(`sql = "${escapeDqlString(ex.sql.trim())}"`);
|
|
3847
|
+
lines.push(` { ${parts.join(', ')} },`);
|
|
3848
|
+
}
|
|
3849
|
+
lines.push(' ]');
|
|
3850
|
+
}
|
|
3851
|
+
}
|
|
3852
|
+
lines.push('');
|
|
3853
|
+
lines.push(' query = """');
|
|
3854
|
+
lines.push(...indentBlock(options.sql.trim(), 8).split('\n'));
|
|
3855
|
+
lines.push(' """');
|
|
3856
|
+
lines.push('');
|
|
3857
|
+
lines.push(' visualization {');
|
|
3858
|
+
lines.push(' chart = "table"');
|
|
3859
|
+
lines.push(' }');
|
|
3860
|
+
lines.push('}');
|
|
3861
|
+
return lines.join('\n') + '\n';
|
|
3862
|
+
}
|
|
3863
|
+
function parseYamlScalar(value) {
|
|
3864
|
+
const trimmed = value.trim();
|
|
3865
|
+
if (!trimmed)
|
|
3866
|
+
return '';
|
|
3867
|
+
if ((trimmed.startsWith('"') && trimmed.endsWith('"'))
|
|
3868
|
+
|| (trimmed.startsWith("'") && trimmed.endsWith("'"))) {
|
|
3869
|
+
return trimmed.slice(1, -1);
|
|
3870
|
+
}
|
|
3871
|
+
return trimmed;
|
|
3872
|
+
}
|
|
3873
|
+
function yamlScalar(value) {
|
|
3874
|
+
if (/^[a-zA-Z0-9_.:/-]+$/.test(value))
|
|
3875
|
+
return value;
|
|
3876
|
+
return JSON.stringify(value);
|
|
3877
|
+
}
|
|
3878
|
+
function buildNotebookTemplate(title, template) {
|
|
3879
|
+
const id = () => Math.random().toString(36).slice(2, 10);
|
|
3880
|
+
let cells;
|
|
3881
|
+
if (template === 'revenue') {
|
|
3882
|
+
cells = [
|
|
3883
|
+
{ id: id(), type: 'markdown', content: `# ${title}\n\nRevenue analysis using DQL and DuckDB.` },
|
|
3884
|
+
{ id: id(), type: 'sql', name: 'revenue_summary', content: "SELECT\n segment_tier AS segment,\n SUM(amount) AS total_revenue,\n COUNT(*) AS deals\nFROM read_csv_auto('./data/revenue.csv')\nGROUP BY segment_tier\nORDER BY total_revenue DESC" },
|
|
3885
|
+
{ id: id(), type: 'sql', name: 'revenue_trend', content: "SELECT\n recognized_at AS date,\n SUM(amount) AS revenue\nFROM read_csv_auto('./data/revenue.csv')\nGROUP BY recognized_at\nORDER BY recognized_at" },
|
|
3886
|
+
];
|
|
3887
|
+
}
|
|
3888
|
+
else if (template === 'pipeline') {
|
|
3889
|
+
cells = [
|
|
3890
|
+
{ id: id(), type: 'markdown', content: `# ${title}\n\nPipeline health and conversion analysis.` },
|
|
3891
|
+
{ id: id(), type: 'sql', name: 'pipeline_overview', content: "SELECT *\nFROM read_csv_auto('./data/pipeline.csv')\nLIMIT 100" },
|
|
3892
|
+
];
|
|
3893
|
+
}
|
|
3894
|
+
else {
|
|
3895
|
+
cells = [
|
|
3896
|
+
{ id: id(), type: 'markdown', content: `# ${title}\n\nAdd your analysis here.` },
|
|
3897
|
+
{ id: id(), type: 'sql', name: 'query_1', content: 'SELECT 1 AS hello' },
|
|
3898
|
+
];
|
|
3899
|
+
}
|
|
3900
|
+
return JSON.stringify({ version: 1, title, cells }, null, 2);
|
|
3901
|
+
}
|
|
3902
|
+
/** Build a lineage graph from the project's blocks and semantic layer. */
|
|
3903
|
+
// Simple lineage graph cache: rebuilds at most every 5 seconds
|
|
3904
|
+
let _lineageCache = null;
|
|
3905
|
+
const LINEAGE_CACHE_TTL_MS = 5000;
|
|
3906
|
+
function buildProjectLineageGraph(projectRoot, semanticLayer) {
|
|
3907
|
+
if (_lineageCache && Date.now() - _lineageCache.builtAt < LINEAGE_CACHE_TTL_MS) {
|
|
3908
|
+
return _lineageCache.graph;
|
|
3909
|
+
}
|
|
3910
|
+
const graph = buildProjectLineageGraphUncached(projectRoot, semanticLayer);
|
|
3911
|
+
_lineageCache = { graph, builtAt: Date.now() };
|
|
3912
|
+
return graph;
|
|
3913
|
+
}
|
|
3914
|
+
function buildProjectLineageGraphUncached(projectRoot, semanticLayer) {
|
|
3915
|
+
const manifestPath = join(projectRoot, 'dql-manifest.json');
|
|
3916
|
+
if (existsSync(manifestPath)) {
|
|
3917
|
+
try {
|
|
3918
|
+
const manifest = JSON.parse(readFileSync(manifestPath, 'utf-8'));
|
|
3919
|
+
if (manifest.lineage?.nodes && manifest.lineage?.edges) {
|
|
3920
|
+
return LineageGraph.fromJSON({
|
|
3921
|
+
nodes: manifest.lineage.nodes,
|
|
3922
|
+
edges: manifest.lineage.edges,
|
|
3923
|
+
});
|
|
3924
|
+
}
|
|
3925
|
+
}
|
|
3926
|
+
catch {
|
|
3927
|
+
// Fall back to a live build.
|
|
3928
|
+
}
|
|
3929
|
+
}
|
|
3930
|
+
const dbtManifestPath = resolveDbtManifestPath(projectRoot);
|
|
3931
|
+
try {
|
|
3932
|
+
const manifest = buildManifest({
|
|
3933
|
+
projectRoot,
|
|
3934
|
+
dbtManifestPath,
|
|
3935
|
+
});
|
|
3936
|
+
return LineageGraph.fromJSON({
|
|
3937
|
+
nodes: manifest.lineage.nodes,
|
|
3938
|
+
edges: manifest.lineage.edges,
|
|
3939
|
+
});
|
|
3940
|
+
}
|
|
3941
|
+
catch {
|
|
3942
|
+
const blocks = [];
|
|
3943
|
+
const metrics = [];
|
|
3944
|
+
const dimensions = [];
|
|
3945
|
+
const dirs = ['blocks', 'dashboards', 'workbooks'];
|
|
3946
|
+
for (const dir of dirs) {
|
|
3947
|
+
const dirPath = join(projectRoot, dir);
|
|
3948
|
+
if (!existsSync(dirPath))
|
|
3949
|
+
continue;
|
|
3950
|
+
for (const entry of readdirSync(dirPath, { withFileTypes: true })) {
|
|
3951
|
+
if (!entry.isFile() || extname(entry.name) !== '.dql')
|
|
3952
|
+
continue;
|
|
3953
|
+
try {
|
|
3954
|
+
const source = readFileSync(join(dirPath, entry.name), 'utf-8');
|
|
3955
|
+
const parser = new Parser(source, `${dir}/${entry.name}`);
|
|
3956
|
+
const ast = parser.parse();
|
|
3957
|
+
for (const stmt of ast.statements) {
|
|
3958
|
+
const block = stmt;
|
|
3959
|
+
if (block.kind !== 'BlockDecl')
|
|
3960
|
+
continue;
|
|
3961
|
+
blocks.push({
|
|
3962
|
+
name: block.name,
|
|
3963
|
+
sql: block.query?.rawSQL ?? '',
|
|
3964
|
+
domain: extractProp(block, 'domain'),
|
|
3965
|
+
owner: extractProp(block, 'owner'),
|
|
3966
|
+
status: extractProp(block, 'status'),
|
|
3967
|
+
blockType: block.blockType,
|
|
3968
|
+
metricRef: block.metricRef,
|
|
3969
|
+
chartType: extractVizChart(block),
|
|
3970
|
+
});
|
|
3971
|
+
}
|
|
3972
|
+
}
|
|
3973
|
+
catch { /* skip unparseable */ }
|
|
3974
|
+
}
|
|
3975
|
+
}
|
|
3976
|
+
if (semanticLayer) {
|
|
3977
|
+
for (const m of semanticLayer.listMetrics()) {
|
|
3978
|
+
metrics.push({ name: m.name, table: m.table, domain: m.domain, type: m.type });
|
|
3979
|
+
}
|
|
3980
|
+
for (const d of semanticLayer.listDimensions()) {
|
|
3981
|
+
dimensions.push({ name: d.name, table: d.table });
|
|
3982
|
+
}
|
|
3983
|
+
}
|
|
3984
|
+
return buildLineageGraph(blocks, metrics, dimensions);
|
|
3985
|
+
}
|
|
3986
|
+
}
|
|
3987
|
+
function resolveDbtManifestPath(projectRoot) {
|
|
3988
|
+
const candidate = join(projectRoot, 'target', 'manifest.json');
|
|
3989
|
+
return existsSync(candidate) ? candidate : undefined;
|
|
3990
|
+
}
|
|
3991
|
+
function resolveLineageNode(graph, rawNodeId) {
|
|
3992
|
+
if (graph.getNode(rawNodeId))
|
|
3993
|
+
return graph.getNode(rawNodeId);
|
|
3994
|
+
const result = queryLineage(graph, { focus: rawNodeId });
|
|
3995
|
+
return result.focalNode;
|
|
3996
|
+
}
|
|
3997
|
+
function extractProp(block, key) {
|
|
3998
|
+
// Check direct AST fields first (parser puts domain, owner, type directly on the node)
|
|
3999
|
+
if (block[key] !== undefined && block[key] !== null)
|
|
4000
|
+
return String(block[key]);
|
|
4001
|
+
for (const prop of block.properties ?? []) {
|
|
4002
|
+
if (prop.key === key && prop.value?.kind === 'Literal')
|
|
4003
|
+
return String(prop.value.value);
|
|
4004
|
+
}
|
|
4005
|
+
return undefined;
|
|
4006
|
+
}
|
|
4007
|
+
function extractVizChart(block) {
|
|
4008
|
+
for (const prop of block.visualization?.properties ?? []) {
|
|
4009
|
+
if (prop.key === 'chart' && prop.value?.kind === 'Literal')
|
|
4010
|
+
return String(prop.value.value);
|
|
4011
|
+
}
|
|
4012
|
+
return undefined;
|
|
4013
|
+
}
|
|
4014
|
+
async function execGit(cwd, args) {
|
|
4015
|
+
const { execFile } = await import('node:child_process');
|
|
4016
|
+
return new Promise((resolve) => {
|
|
4017
|
+
execFile('git', args, { cwd, maxBuffer: 8 * 1024 * 1024 }, (err, stdout, stderr) => {
|
|
4018
|
+
resolve({
|
|
4019
|
+
stdout: String(stdout ?? ''),
|
|
4020
|
+
stderr: String(stderr ?? ''),
|
|
4021
|
+
code: err ? (err.code ? 1 : err.code ?? 1) : 0,
|
|
4022
|
+
});
|
|
4023
|
+
});
|
|
4024
|
+
});
|
|
4025
|
+
}
|
|
4026
|
+
async function readGitStatus(cwd) {
|
|
4027
|
+
const isRepo = await execGit(cwd, ['rev-parse', '--is-inside-work-tree']);
|
|
4028
|
+
if (isRepo.code !== 0 || isRepo.stdout.trim() !== 'true') {
|
|
4029
|
+
return { inRepo: false, branch: null, ahead: 0, behind: 0, changes: [] };
|
|
4030
|
+
}
|
|
4031
|
+
const branchRes = await execGit(cwd, ['rev-parse', '--abbrev-ref', 'HEAD']);
|
|
4032
|
+
const branch = branchRes.code === 0 ? branchRes.stdout.trim() : null;
|
|
4033
|
+
const trackRes = await execGit(cwd, ['rev-list', '--left-right', '--count', '@{u}...HEAD']);
|
|
4034
|
+
let ahead = 0;
|
|
4035
|
+
let behind = 0;
|
|
4036
|
+
if (trackRes.code === 0) {
|
|
4037
|
+
const match = trackRes.stdout.trim().split(/\s+/);
|
|
4038
|
+
behind = Number(match[0] ?? 0);
|
|
4039
|
+
ahead = Number(match[1] ?? 0);
|
|
4040
|
+
}
|
|
4041
|
+
const statusRes = await execGit(cwd, ['status', '--porcelain=v1', '--untracked-files=normal']);
|
|
4042
|
+
const changes = [];
|
|
4043
|
+
if (statusRes.code === 0) {
|
|
4044
|
+
for (const line of statusRes.stdout.split('\n')) {
|
|
4045
|
+
if (!line)
|
|
4046
|
+
continue;
|
|
4047
|
+
const code = line.slice(0, 2);
|
|
4048
|
+
const p = line.slice(3);
|
|
4049
|
+
changes.push({ path: p, status: code });
|
|
4050
|
+
}
|
|
4051
|
+
}
|
|
4052
|
+
return { inRepo: true, branch, ahead, behind, changes };
|
|
4053
|
+
}
|
|
4054
|
+
async function readGitLog(cwd, limit) {
|
|
4055
|
+
const isRepo = await execGit(cwd, ['rev-parse', '--is-inside-work-tree']);
|
|
4056
|
+
if (isRepo.code !== 0)
|
|
4057
|
+
return { inRepo: false, commits: [] };
|
|
4058
|
+
const sep = '\x1f';
|
|
4059
|
+
const end = '\x1e';
|
|
4060
|
+
const fmt = ['%H', '%an', '%ad', '%s'].join(sep) + end;
|
|
4061
|
+
const res = await execGit(cwd, ['log', `-${limit}`, `--pretty=format:${fmt}`, '--date=short']);
|
|
4062
|
+
if (res.code !== 0)
|
|
4063
|
+
return { inRepo: true, commits: [] };
|
|
4064
|
+
const commits = [];
|
|
4065
|
+
for (const entry of res.stdout.split(end)) {
|
|
4066
|
+
const trimmed = entry.replace(/^\n/, '');
|
|
4067
|
+
if (!trimmed)
|
|
4068
|
+
continue;
|
|
4069
|
+
const [hash, author, date, subject] = trimmed.split(sep);
|
|
4070
|
+
if (hash)
|
|
4071
|
+
commits.push({ hash, author, date, subject });
|
|
4072
|
+
}
|
|
4073
|
+
return { inRepo: true, commits };
|
|
4074
|
+
}
|
|
4075
|
+
function snapshotPathFor(projectRoot, notebookPath) {
|
|
4076
|
+
const abs = safeJoin(projectRoot, notebookPath);
|
|
4077
|
+
if (!abs)
|
|
4078
|
+
return null;
|
|
4079
|
+
// Strip extension and append `.run.json` so `foo.dqlnb` → `foo.run.json`
|
|
4080
|
+
// and `bar.dql` → `bar.run.json`. Keeps the sibling file next to source.
|
|
4081
|
+
const dot = abs.lastIndexOf('.');
|
|
4082
|
+
const base = dot > abs.lastIndexOf('/') ? abs.slice(0, dot) : abs;
|
|
4083
|
+
return `${base}.run.json`;
|
|
4084
|
+
}
|
|
4085
|
+
function readRunSnapshot(projectRoot, notebookPath) {
|
|
4086
|
+
const p = snapshotPathFor(projectRoot, notebookPath);
|
|
4087
|
+
if (!p || !existsSync(p))
|
|
4088
|
+
return { found: false, snapshot: null };
|
|
4089
|
+
try {
|
|
4090
|
+
const raw = readFileSync(p, 'utf-8');
|
|
4091
|
+
return { found: true, snapshot: JSON.parse(raw) };
|
|
4092
|
+
}
|
|
4093
|
+
catch {
|
|
4094
|
+
return { found: false, snapshot: null };
|
|
4095
|
+
}
|
|
4096
|
+
}
|
|
4097
|
+
function writeRunSnapshot(projectRoot, notebookPath, snapshot) {
|
|
4098
|
+
const p = snapshotPathFor(projectRoot, notebookPath);
|
|
4099
|
+
if (!p)
|
|
4100
|
+
throw new Error('Invalid path');
|
|
4101
|
+
mkdirSync(dirname(p), { recursive: true });
|
|
4102
|
+
writeFileSync(p, JSON.stringify(snapshot, null, 2), 'utf-8');
|
|
4103
|
+
// Append `*.run.json` to .gitignore once, so snapshots don't pollute git
|
|
4104
|
+
// history unless the user deliberately un-ignores them.
|
|
4105
|
+
ensureGitignoreEntry(projectRoot, '*.run.json');
|
|
4106
|
+
}
|
|
4107
|
+
function ensureGitignoreEntry(projectRoot, pattern) {
|
|
4108
|
+
try {
|
|
4109
|
+
const gitignorePath = join(projectRoot, '.gitignore');
|
|
4110
|
+
const existing = existsSync(gitignorePath) ? readFileSync(gitignorePath, 'utf-8') : '';
|
|
4111
|
+
const lines = existing.split('\n').map((l) => l.trim());
|
|
4112
|
+
if (lines.includes(pattern))
|
|
4113
|
+
return;
|
|
4114
|
+
const next = existing.endsWith('\n') || existing === ''
|
|
4115
|
+
? `${existing}${pattern}\n`
|
|
4116
|
+
: `${existing}\n${pattern}\n`;
|
|
4117
|
+
writeFileSync(gitignorePath, next, 'utf-8');
|
|
4118
|
+
}
|
|
4119
|
+
catch {
|
|
4120
|
+
// Best-effort; failure to write .gitignore shouldn't fail the snapshot.
|
|
4121
|
+
}
|
|
4122
|
+
}
|
|
4123
|
+
async function readGitDiff(cwd, filePath, staged = false) {
|
|
4124
|
+
const isRepo = await execGit(cwd, ['rev-parse', '--is-inside-work-tree']);
|
|
4125
|
+
if (isRepo.code !== 0) {
|
|
4126
|
+
return { inRepo: false, diff: '', before: null, after: null, diffReport: null };
|
|
4127
|
+
}
|
|
4128
|
+
const baseArgs = staged ? ['diff', '--cached', '--no-color'] : ['diff', '--no-color'];
|
|
4129
|
+
if (!filePath) {
|
|
4130
|
+
const res = await execGit(cwd, baseArgs);
|
|
4131
|
+
return { inRepo: true, diff: res.stdout, before: null, after: null, diffReport: null };
|
|
4132
|
+
}
|
|
4133
|
+
const isSemantic = filePath.endsWith('.dql') || filePath.endsWith('.dqlnb');
|
|
4134
|
+
const [diffRes, before, after] = await Promise.all([
|
|
4135
|
+
execGit(cwd, [...baseArgs, '--', filePath]),
|
|
4136
|
+
isSemantic ? readHeadBlob(cwd, filePath) : Promise.resolve(null),
|
|
4137
|
+
isSemantic ? readWorkingCopy(join(cwd, filePath)) : Promise.resolve(null),
|
|
4138
|
+
]);
|
|
4139
|
+
const diffReport = isSemantic ? computeSemanticDiff(filePath, before, after) : null;
|
|
4140
|
+
return { inRepo: true, diff: diffRes.stdout, before, after, diffReport };
|
|
4141
|
+
}
|
|
4142
|
+
// ── git write operations ──────────────────────────────────────────────────
|
|
4143
|
+
// Each helper validates inputs, shells out via execFile (no shell expansion),
|
|
4144
|
+
// then reports the trimmed stderr on failure so the UI can surface it. We
|
|
4145
|
+
// never accept absolute paths or paths containing `..` — staged paths must
|
|
4146
|
+
// stay inside the project root.
|
|
4147
|
+
function validatePaths(cwd, paths) {
|
|
4148
|
+
if (!Array.isArray(paths) || paths.length === 0) {
|
|
4149
|
+
return { ok: false, error: 'No paths provided' };
|
|
4150
|
+
}
|
|
4151
|
+
const cleaned = [];
|
|
4152
|
+
for (const p of paths) {
|
|
4153
|
+
if (typeof p !== 'string' || p.length === 0)
|
|
4154
|
+
return { ok: false, error: 'Invalid path' };
|
|
4155
|
+
if (p.startsWith('/'))
|
|
4156
|
+
return { ok: false, error: `Absolute path not allowed: ${p}` };
|
|
4157
|
+
if (p.split('/').includes('..'))
|
|
4158
|
+
return { ok: false, error: `Path escape not allowed: ${p}` };
|
|
4159
|
+
const resolved = join(cwd, p);
|
|
4160
|
+
if (!resolved.startsWith(cwd))
|
|
4161
|
+
return { ok: false, error: `Path outside project: ${p}` };
|
|
4162
|
+
cleaned.push(p);
|
|
4163
|
+
}
|
|
4164
|
+
return { ok: true, paths: cleaned };
|
|
4165
|
+
}
|
|
4166
|
+
function gitErrorOutput(res) {
|
|
4167
|
+
return (res.stderr || res.stdout || '').trim();
|
|
4168
|
+
}
|
|
4169
|
+
async function gitStage(cwd, paths) {
|
|
4170
|
+
const v = validatePaths(cwd, paths);
|
|
4171
|
+
if (!v.ok)
|
|
4172
|
+
return { ok: false, error: v.error };
|
|
4173
|
+
const res = await execGit(cwd, ['add', '--', ...v.paths]);
|
|
4174
|
+
return res.code === 0 ? { ok: true } : { ok: false, error: gitErrorOutput(res) };
|
|
4175
|
+
}
|
|
4176
|
+
async function gitUnstage(cwd, paths) {
|
|
4177
|
+
const v = validatePaths(cwd, paths);
|
|
4178
|
+
if (!v.ok)
|
|
4179
|
+
return { ok: false, error: v.error };
|
|
4180
|
+
// `restore --staged` works with or without HEAD; for an initial commit (no
|
|
4181
|
+
// HEAD yet) git's `rm --cached` is the fallback. Try restore first.
|
|
4182
|
+
const res = await execGit(cwd, ['restore', '--staged', '--', ...v.paths]);
|
|
4183
|
+
if (res.code === 0)
|
|
4184
|
+
return { ok: true };
|
|
4185
|
+
const fallback = await execGit(cwd, ['rm', '--cached', '-r', '--', ...v.paths]);
|
|
4186
|
+
return fallback.code === 0 ? { ok: true } : { ok: false, error: gitErrorOutput(fallback) };
|
|
4187
|
+
}
|
|
4188
|
+
async function gitDiscard(cwd, paths) {
|
|
4189
|
+
const v = validatePaths(cwd, paths);
|
|
4190
|
+
if (!v.ok)
|
|
4191
|
+
return { ok: false, error: v.error };
|
|
4192
|
+
// For tracked files: `restore --worktree` reverts to HEAD. For untracked
|
|
4193
|
+
// files: that's a no-op and we delete them via `clean -f`. Run both so
|
|
4194
|
+
// the caller doesn't have to know which list each path is in.
|
|
4195
|
+
const restore = await execGit(cwd, ['restore', '--worktree', '--', ...v.paths]);
|
|
4196
|
+
const clean = await execGit(cwd, ['clean', '-f', '--', ...v.paths]);
|
|
4197
|
+
if (restore.code !== 0 && clean.code !== 0) {
|
|
4198
|
+
return { ok: false, error: gitErrorOutput(restore) || gitErrorOutput(clean) };
|
|
4199
|
+
}
|
|
4200
|
+
return { ok: true };
|
|
4201
|
+
}
|
|
4202
|
+
async function gitCommit(cwd, message, stageAll) {
|
|
4203
|
+
const trimmed = message.trim();
|
|
4204
|
+
if (!trimmed)
|
|
4205
|
+
return { ok: false, error: 'Commit message required' };
|
|
4206
|
+
if (stageAll) {
|
|
4207
|
+
const add = await execGit(cwd, ['add', '-A']);
|
|
4208
|
+
if (add.code !== 0)
|
|
4209
|
+
return { ok: false, error: gitErrorOutput(add) };
|
|
4210
|
+
}
|
|
4211
|
+
const res = await execGit(cwd, ['commit', '-m', trimmed]);
|
|
4212
|
+
if (res.code !== 0)
|
|
4213
|
+
return { ok: false, error: gitErrorOutput(res) };
|
|
4214
|
+
const hashRes = await execGit(cwd, ['rev-parse', 'HEAD']);
|
|
4215
|
+
return { ok: true, hash: hashRes.code === 0 ? hashRes.stdout.trim() : undefined };
|
|
4216
|
+
}
|
|
4217
|
+
async function gitPush(cwd) {
|
|
4218
|
+
const res = await execGit(cwd, ['push']);
|
|
4219
|
+
return res.code === 0
|
|
4220
|
+
? { ok: true, output: gitErrorOutput(res) }
|
|
4221
|
+
: { ok: false, error: gitErrorOutput(res) };
|
|
4222
|
+
}
|
|
4223
|
+
async function gitPull(cwd) {
|
|
4224
|
+
// `--ff-only` keeps the operation non-destructive: if the local branch has
|
|
4225
|
+
// diverged from upstream, we surface the error rather than auto-merging.
|
|
4226
|
+
// The user can resolve via the terminal or a future merge UI.
|
|
4227
|
+
const res = await execGit(cwd, ['pull', '--ff-only']);
|
|
4228
|
+
return res.code === 0
|
|
4229
|
+
? { ok: true, output: gitErrorOutput(res) }
|
|
4230
|
+
: { ok: false, error: gitErrorOutput(res) };
|
|
4231
|
+
}
|
|
4232
|
+
async function readGitBranches(cwd) {
|
|
4233
|
+
const isRepo = await execGit(cwd, ['rev-parse', '--is-inside-work-tree']);
|
|
4234
|
+
if (isRepo.code !== 0)
|
|
4235
|
+
return { inRepo: false, current: null, branches: [] };
|
|
4236
|
+
const cur = await execGit(cwd, ['rev-parse', '--abbrev-ref', 'HEAD']);
|
|
4237
|
+
const list = await execGit(cwd, ['branch', '--list', '--format=%(refname:short)']);
|
|
4238
|
+
const branches = list.code === 0
|
|
4239
|
+
? list.stdout.split('\n').map((s) => s.trim()).filter(Boolean)
|
|
4240
|
+
: [];
|
|
4241
|
+
return { inRepo: true, current: cur.code === 0 ? cur.stdout.trim() : null, branches };
|
|
4242
|
+
}
|
|
4243
|
+
async function readGitRemote(cwd) {
|
|
4244
|
+
const isRepo = await execGit(cwd, ['rev-parse', '--is-inside-work-tree']);
|
|
4245
|
+
if (isRepo.code !== 0)
|
|
4246
|
+
return { inRepo: false, url: null, name: null };
|
|
4247
|
+
const remoteName = await execGit(cwd, ['config', '--get', 'remote.pushDefault']);
|
|
4248
|
+
const name = remoteName.code === 0 && remoteName.stdout.trim() ? remoteName.stdout.trim() : 'origin';
|
|
4249
|
+
const url = await execGit(cwd, ['remote', 'get-url', name]);
|
|
4250
|
+
return { inRepo: true, url: url.code === 0 ? url.stdout.trim() : null, name };
|
|
4251
|
+
}
|
|
4252
|
+
async function gitCreateBranch(cwd, name, checkout) {
|
|
4253
|
+
const trimmed = name.trim();
|
|
4254
|
+
// Branch names can't start with `-` (would be parsed as a flag) and must be
|
|
4255
|
+
// non-empty. git itself enforces the rest of the ref-name rules.
|
|
4256
|
+
if (!trimmed)
|
|
4257
|
+
return { ok: false, error: 'Branch name required' };
|
|
4258
|
+
if (trimmed.startsWith('-'))
|
|
4259
|
+
return { ok: false, error: 'Invalid branch name' };
|
|
4260
|
+
const res = checkout
|
|
4261
|
+
? await execGit(cwd, ['checkout', '-b', trimmed])
|
|
4262
|
+
: await execGit(cwd, ['branch', trimmed]);
|
|
4263
|
+
return res.code === 0 ? { ok: true } : { ok: false, error: gitErrorOutput(res) };
|
|
4264
|
+
}
|
|
4265
|
+
async function gitCheckout(cwd, name) {
|
|
4266
|
+
const trimmed = name.trim();
|
|
4267
|
+
if (!trimmed)
|
|
4268
|
+
return { ok: false, error: 'Branch name required' };
|
|
4269
|
+
if (trimmed.startsWith('-'))
|
|
4270
|
+
return { ok: false, error: 'Invalid branch name' };
|
|
4271
|
+
const res = await execGit(cwd, ['checkout', trimmed]);
|
|
4272
|
+
return res.code === 0 ? { ok: true } : { ok: false, error: gitErrorOutput(res) };
|
|
4273
|
+
}
|
|
4274
|
+
async function readHeadBlob(cwd, filePath) {
|
|
4275
|
+
try {
|
|
4276
|
+
const res = await execGit(cwd, ['show', `HEAD:${filePath}`]);
|
|
4277
|
+
return res.code === 0 ? res.stdout : null;
|
|
4278
|
+
}
|
|
4279
|
+
catch {
|
|
4280
|
+
return null;
|
|
4281
|
+
}
|
|
4282
|
+
}
|
|
4283
|
+
async function readWorkingCopy(absPath) {
|
|
4284
|
+
try {
|
|
4285
|
+
return readFileSync(absPath, 'utf-8');
|
|
4286
|
+
}
|
|
4287
|
+
catch {
|
|
4288
|
+
return null;
|
|
4289
|
+
}
|
|
4290
|
+
}
|
|
4291
|
+
function computeSemanticDiff(filePath, before, after) {
|
|
4292
|
+
if (before === after)
|
|
4293
|
+
return null;
|
|
4294
|
+
try {
|
|
4295
|
+
return filePath.endsWith('.dqlnb')
|
|
4296
|
+
? diffNotebook(before, after)
|
|
4297
|
+
: diffDQL(before ?? '', after ?? '');
|
|
4298
|
+
}
|
|
4299
|
+
catch {
|
|
4300
|
+
return null;
|
|
4301
|
+
}
|
|
4302
|
+
}
|
|
4303
|
+
function collectSettingsEnvStatus() {
|
|
4304
|
+
const v = (key, label, description, optional = true) => ({
|
|
4305
|
+
key,
|
|
4306
|
+
label,
|
|
4307
|
+
description,
|
|
4308
|
+
optional,
|
|
4309
|
+
present: typeof process.env[key] === 'string' && process.env[key].trim().length > 0,
|
|
4310
|
+
});
|
|
4311
|
+
return [
|
|
4312
|
+
{
|
|
4313
|
+
id: 'ai',
|
|
4314
|
+
title: 'AI Chat Providers',
|
|
4315
|
+
description: 'Configure one or more providers. Missing keys are only a problem when that provider is selected.',
|
|
4316
|
+
vars: [
|
|
4317
|
+
v('ANTHROPIC_API_KEY', 'Claude Agent SDK', 'Hosted Claude provider for notebook Chat and agent commands.'),
|
|
4318
|
+
v('OPENAI_API_KEY', 'OpenAI', 'Hosted OpenAI provider for notebook Chat and agent commands.'),
|
|
4319
|
+
v('OPENAI_MODEL', 'OpenAI model', 'Optional override such as gpt-4.1-mini or the model your account uses.'),
|
|
4320
|
+
v('GEMINI_API_KEY', 'Gemini', 'Hosted Gemini provider for notebook Chat and agent commands.'),
|
|
4321
|
+
v('GEMINI_MODEL', 'Gemini model', 'Optional Gemini model override.'),
|
|
4322
|
+
v('OLLAMA_BASE_URL', 'Ollama base URL', 'Local Ollama HTTP endpoint. Docker defaults to http://ollama:11434.'),
|
|
4323
|
+
v('OLLAMA_MODEL', 'Ollama model', 'Optional local model name such as llama3.1.'),
|
|
4324
|
+
],
|
|
4325
|
+
},
|
|
4326
|
+
{
|
|
4327
|
+
id: 'slack',
|
|
4328
|
+
title: 'Slack',
|
|
4329
|
+
description: 'Use webhooks for scheduled App deliveries, or bot credentials for the Slack chat front-end.',
|
|
4330
|
+
vars: [
|
|
4331
|
+
v('DQL_SLACK_WEBHOOK', 'Schedule webhook', 'Incoming webhook used by App schedules that deliver to Slack.'),
|
|
4332
|
+
v('SLACK_SIGNING_SECRET', 'Slack signing secret', 'Required only when running `dql slack serve`.'),
|
|
4333
|
+
v('SLACK_BOT_TOKEN', 'Slack bot token', 'Bot token used by Slack chat commands when `dql slack serve` is enabled.'),
|
|
4334
|
+
],
|
|
4335
|
+
},
|
|
4336
|
+
{
|
|
4337
|
+
id: 'email',
|
|
4338
|
+
title: 'Email',
|
|
4339
|
+
description: 'SMTP is optional. Without it, email schedules stay in stub mode with a clear delivery message.',
|
|
4340
|
+
vars: [
|
|
4341
|
+
v('DQL_SMTP_URL', 'SMTP URL', 'SMTP connection URL for email schedule delivery.'),
|
|
4342
|
+
v('DQL_SMTP_FROM', 'SMTP sender', 'Optional sender address for scheduled emails.'),
|
|
4343
|
+
],
|
|
4344
|
+
},
|
|
4345
|
+
{
|
|
4346
|
+
id: 'runtime',
|
|
4347
|
+
title: 'Runtime',
|
|
4348
|
+
description: 'Local server and runtime toggles used by Docker and native notebook sessions.',
|
|
4349
|
+
vars: [
|
|
4350
|
+
v('DQL_HOST', 'Notebook bind host', 'Host interface for the local notebook server. Docker uses 0.0.0.0 inside the container.'),
|
|
4351
|
+
v('DQL_RUNTIME_URL', 'Runtime URL', 'Optional URL used by headless agent commands to call an existing runtime.'),
|
|
4352
|
+
v('DQL_LLM_KEY', 'Legacy LLM key', 'Fallback key accepted by older Claude provider configurations.'),
|
|
4353
|
+
],
|
|
4354
|
+
},
|
|
4355
|
+
];
|
|
4356
|
+
}
|
|
4357
|
+
//# sourceMappingURL=local-runtime.js.map
|