@deepagents/text2sql 5.0.0 → 6.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLICKHOUSE-DEPLOYMENT.md +181 -0
- package/README.md +26 -4
- package/dist/bin/sql.js +33 -51
- package/dist/index.js +56 -289
- package/dist/index.js.map +4 -4
- package/dist/lib/adapters/adapter.d.ts +8 -1
- package/dist/lib/adapters/adapter.d.ts.map +1 -1
- package/dist/lib/adapters/bigquery/bigquery.d.ts.map +1 -1
- package/dist/lib/adapters/bigquery/bigquery.sql-policy.d.ts +9 -0
- package/dist/lib/adapters/bigquery/bigquery.sql-policy.d.ts.map +1 -0
- package/dist/lib/adapters/bigquery/index.d.ts +1 -0
- package/dist/lib/adapters/bigquery/index.d.ts.map +1 -1
- package/dist/lib/adapters/bigquery/index.js +303 -259
- package/dist/lib/adapters/bigquery/index.js.map +4 -4
- package/dist/lib/adapters/clickhouse/clickhouse.d.ts +22 -0
- package/dist/lib/adapters/clickhouse/clickhouse.d.ts.map +1 -0
- package/dist/lib/adapters/clickhouse/clickhouse.sql-policy.d.ts +8 -0
- package/dist/lib/adapters/clickhouse/clickhouse.sql-policy.d.ts.map +1 -0
- package/dist/lib/adapters/clickhouse/constraint.clickhouse.grounding.d.ts +8 -0
- package/dist/lib/adapters/clickhouse/constraint.clickhouse.grounding.d.ts.map +1 -0
- package/dist/lib/adapters/clickhouse/index.d.ts +33 -0
- package/dist/lib/adapters/clickhouse/index.d.ts.map +1 -0
- package/dist/lib/adapters/clickhouse/index.js +1544 -0
- package/dist/lib/adapters/clickhouse/index.js.map +7 -0
- package/dist/lib/adapters/clickhouse/indexes.clickhouse.grounding.d.ts +8 -0
- package/dist/lib/adapters/clickhouse/indexes.clickhouse.grounding.d.ts.map +1 -0
- package/dist/lib/adapters/clickhouse/info.clickhouse.grounding.d.ts +8 -0
- package/dist/lib/adapters/clickhouse/info.clickhouse.grounding.d.ts.map +1 -0
- package/dist/lib/adapters/clickhouse/row-count.clickhouse.grounding.d.ts +8 -0
- package/dist/lib/adapters/clickhouse/row-count.clickhouse.grounding.d.ts.map +1 -0
- package/dist/lib/adapters/clickhouse/table.clickhouse.grounding.d.ts +11 -0
- package/dist/lib/adapters/clickhouse/table.clickhouse.grounding.d.ts.map +1 -0
- package/dist/lib/adapters/clickhouse/view.clickhouse.grounding.d.ts +9 -0
- package/dist/lib/adapters/clickhouse/view.clickhouse.grounding.d.ts.map +1 -0
- package/dist/lib/adapters/groundings/abstract.grounding.d.ts +11 -16
- package/dist/lib/adapters/groundings/abstract.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/groundings/index.js +16 -0
- package/dist/lib/adapters/groundings/index.js.map +2 -2
- package/dist/lib/adapters/groundings/table.grounding.d.ts +1 -0
- package/dist/lib/adapters/groundings/table.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/groundings/view.grounding.d.ts +1 -0
- package/dist/lib/adapters/groundings/view.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/mysql/index.d.ts +1 -0
- package/dist/lib/adapters/mysql/index.d.ts.map +1 -1
- package/dist/lib/adapters/mysql/index.js +317 -259
- package/dist/lib/adapters/mysql/index.js.map +4 -4
- package/dist/lib/adapters/mysql/mysql.d.ts.map +1 -1
- package/dist/lib/adapters/mysql/mysql.sql-policy.d.ts +11 -0
- package/dist/lib/adapters/mysql/mysql.sql-policy.d.ts.map +1 -0
- package/dist/lib/adapters/parser-sql-policy.d.ts +23 -0
- package/dist/lib/adapters/parser-sql-policy.d.ts.map +1 -0
- package/dist/lib/adapters/postgres/index.d.ts +1 -0
- package/dist/lib/adapters/postgres/index.d.ts.map +1 -1
- package/dist/lib/adapters/postgres/index.js +323 -259
- package/dist/lib/adapters/postgres/index.js.map +4 -4
- package/dist/lib/adapters/postgres/postgres.d.ts.map +1 -1
- package/dist/lib/adapters/postgres/postgres.sql-policy.d.ts +10 -0
- package/dist/lib/adapters/postgres/postgres.sql-policy.d.ts.map +1 -0
- package/dist/lib/adapters/spreadsheet/index.js +301 -259
- package/dist/lib/adapters/spreadsheet/index.js.map +4 -4
- package/dist/lib/adapters/spreadsheet/parser.d.ts.map +1 -1
- package/dist/lib/adapters/sql-policy.d.ts +19 -0
- package/dist/lib/adapters/sql-policy.d.ts.map +1 -0
- package/dist/lib/adapters/sqlite/index.d.ts +1 -0
- package/dist/lib/adapters/sqlite/index.d.ts.map +1 -1
- package/dist/lib/adapters/sqlite/index.js +303 -259
- package/dist/lib/adapters/sqlite/index.js.map +4 -4
- package/dist/lib/adapters/sqlite/sqlite.d.ts.map +1 -1
- package/dist/lib/adapters/sqlite/sqlite.sql-policy.d.ts +8 -0
- package/dist/lib/adapters/sqlite/sqlite.sql-policy.d.ts.map +1 -0
- package/dist/lib/adapters/sqlserver/index.d.ts +1 -0
- package/dist/lib/adapters/sqlserver/index.d.ts.map +1 -1
- package/dist/lib/adapters/sqlserver/index.js +310 -259
- package/dist/lib/adapters/sqlserver/index.js.map +4 -4
- package/dist/lib/adapters/sqlserver/sqlserver.d.ts.map +1 -1
- package/dist/lib/adapters/sqlserver/sqlserver.sql-policy.d.ts +11 -0
- package/dist/lib/adapters/sqlserver/sqlserver.sql-policy.d.ts.map +1 -0
- package/dist/lib/agents/exceptions.d.ts +1 -1
- package/dist/lib/agents/sql.agent.d.ts.map +1 -1
- package/dist/lib/sql-scope-error.d.ts +3 -0
- package/dist/lib/sql-scope-error.d.ts.map +1 -0
- package/dist/lib/synthesis/index.js +32 -38
- package/dist/lib/synthesis/index.js.map +4 -4
- package/package.json +29 -23
- package/dist/lib/adapters/runtime-scope.d.ts +0 -16
- package/dist/lib/adapters/runtime-scope.d.ts.map +0 -1
|
@@ -0,0 +1,1544 @@
|
|
|
1
|
+
// packages/text2sql/src/lib/adapters/adapter.ts
|
|
2
|
+
import { format as formatSql } from "sql-formatter";
|
|
3
|
+
|
|
4
|
+
// packages/text2sql/src/lib/agents/exceptions.ts
|
|
5
|
+
var sqlValidationMarker = /* @__PURE__ */ Symbol("SQLValidationError");
|
|
6
|
+
var unanswerableSqlMarker = /* @__PURE__ */ Symbol("UnanswerableSQLError");
|
|
7
|
+
var sqlScopeMarker = /* @__PURE__ */ Symbol("SQLScopeError");
|
|
8
|
+
var sqlReadOnlyMarker = /* @__PURE__ */ Symbol("SQLReadOnlyError");
|
|
9
|
+
var SQLValidationError = class _SQLValidationError extends Error {
|
|
10
|
+
[sqlValidationMarker];
|
|
11
|
+
constructor(message) {
|
|
12
|
+
super(message);
|
|
13
|
+
this.name = "SQLValidationError";
|
|
14
|
+
this[sqlValidationMarker] = true;
|
|
15
|
+
}
|
|
16
|
+
static isInstance(error) {
|
|
17
|
+
return error instanceof _SQLValidationError && error[sqlValidationMarker] === true;
|
|
18
|
+
}
|
|
19
|
+
};
|
|
20
|
+
var UnanswerableSQLError = class _UnanswerableSQLError extends Error {
|
|
21
|
+
[unanswerableSqlMarker];
|
|
22
|
+
constructor(message) {
|
|
23
|
+
super(message);
|
|
24
|
+
this.name = "UnanswerableSQLError";
|
|
25
|
+
this[unanswerableSqlMarker] = true;
|
|
26
|
+
}
|
|
27
|
+
static isInstance(error) {
|
|
28
|
+
return error instanceof _UnanswerableSQLError && error[unanswerableSqlMarker] === true;
|
|
29
|
+
}
|
|
30
|
+
};
|
|
31
|
+
var SQLScopeError = class _SQLScopeError extends Error {
|
|
32
|
+
[sqlScopeMarker];
|
|
33
|
+
payload;
|
|
34
|
+
errorType;
|
|
35
|
+
constructor(payload) {
|
|
36
|
+
super(JSON.stringify(payload));
|
|
37
|
+
this.name = "SQLScopeError";
|
|
38
|
+
this.payload = payload;
|
|
39
|
+
this.errorType = payload.error_type;
|
|
40
|
+
this[sqlScopeMarker] = true;
|
|
41
|
+
}
|
|
42
|
+
static isInstance(error) {
|
|
43
|
+
return error instanceof _SQLScopeError && error[sqlScopeMarker] === true;
|
|
44
|
+
}
|
|
45
|
+
};
|
|
46
|
+
var SQLReadOnlyError = class _SQLReadOnlyError extends Error {
|
|
47
|
+
[sqlReadOnlyMarker];
|
|
48
|
+
constructor(message) {
|
|
49
|
+
super(message);
|
|
50
|
+
this.name = "SQLReadOnlyError";
|
|
51
|
+
this[sqlReadOnlyMarker] = true;
|
|
52
|
+
}
|
|
53
|
+
static isInstance(error) {
|
|
54
|
+
return error instanceof _SQLReadOnlyError && error[sqlReadOnlyMarker] === true;
|
|
55
|
+
}
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
// packages/text2sql/src/lib/fragments/schema.ts
|
|
59
|
+
function dialectInfo(input) {
|
|
60
|
+
return {
|
|
61
|
+
name: "dialectInfo",
|
|
62
|
+
data: {
|
|
63
|
+
dialect: input.dialect,
|
|
64
|
+
...input.version && { version: input.version },
|
|
65
|
+
...input.database && { database: input.database },
|
|
66
|
+
...input.details && { details: input.details }
|
|
67
|
+
}
|
|
68
|
+
};
|
|
69
|
+
}
|
|
70
|
+
function table(input) {
|
|
71
|
+
return {
|
|
72
|
+
name: "table",
|
|
73
|
+
data: {
|
|
74
|
+
name: input.name,
|
|
75
|
+
...input.schema && { schema: input.schema },
|
|
76
|
+
...input.rowCount != null && { rowCount: input.rowCount },
|
|
77
|
+
...input.sizeHint && { sizeHint: input.sizeHint },
|
|
78
|
+
columns: input.columns,
|
|
79
|
+
...input.indexes?.length && { indexes: input.indexes },
|
|
80
|
+
...input.constraints?.length && { constraints: input.constraints }
|
|
81
|
+
}
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
function column(input) {
|
|
85
|
+
return {
|
|
86
|
+
name: "column",
|
|
87
|
+
data: {
|
|
88
|
+
name: input.name,
|
|
89
|
+
type: input.type,
|
|
90
|
+
...input.pk && { pk: true },
|
|
91
|
+
...input.fk && { fk: input.fk },
|
|
92
|
+
...input.unique && { unique: true },
|
|
93
|
+
...input.notNull && { notNull: true },
|
|
94
|
+
...input.default && { default: input.default },
|
|
95
|
+
...input.indexed && { indexed: true },
|
|
96
|
+
...input.values?.length && { values: input.values },
|
|
97
|
+
...input.stats && { stats: input.stats }
|
|
98
|
+
}
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
function index(input) {
|
|
102
|
+
return {
|
|
103
|
+
name: "index",
|
|
104
|
+
data: {
|
|
105
|
+
name: input.name,
|
|
106
|
+
columns: input.columns,
|
|
107
|
+
...input.unique && { unique: true },
|
|
108
|
+
...input.type && { type: input.type }
|
|
109
|
+
}
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
function constraint(input) {
|
|
113
|
+
return {
|
|
114
|
+
name: "constraint",
|
|
115
|
+
data: {
|
|
116
|
+
name: input.name,
|
|
117
|
+
type: input.type,
|
|
118
|
+
...input.columns?.length && { columns: input.columns },
|
|
119
|
+
...input.definition && { definition: input.definition },
|
|
120
|
+
...input.defaultValue && { defaultValue: input.defaultValue },
|
|
121
|
+
...input.referencedTable && { referencedTable: input.referencedTable },
|
|
122
|
+
...input.referencedColumns?.length && {
|
|
123
|
+
referencedColumns: input.referencedColumns
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
function view(input) {
|
|
129
|
+
return {
|
|
130
|
+
name: "view",
|
|
131
|
+
data: {
|
|
132
|
+
name: input.name,
|
|
133
|
+
...input.schema && { schema: input.schema },
|
|
134
|
+
columns: input.columns,
|
|
135
|
+
...input.definition && { definition: input.definition }
|
|
136
|
+
}
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
function relationship(input) {
|
|
140
|
+
return {
|
|
141
|
+
name: "relationship",
|
|
142
|
+
data: {
|
|
143
|
+
from: input.from,
|
|
144
|
+
to: input.to,
|
|
145
|
+
...input.cardinality && { cardinality: input.cardinality }
|
|
146
|
+
}
|
|
147
|
+
};
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
// packages/text2sql/src/lib/adapters/groundings/context.ts
|
|
151
|
+
var noopProgress = () => {
|
|
152
|
+
};
|
|
153
|
+
function timestampProgress(onProgress) {
|
|
154
|
+
if (!onProgress) return noopProgress;
|
|
155
|
+
return (progress) => {
|
|
156
|
+
onProgress({
|
|
157
|
+
...progress,
|
|
158
|
+
timestampMs: progress.timestampMs ?? Date.now()
|
|
159
|
+
});
|
|
160
|
+
};
|
|
161
|
+
}
|
|
162
|
+
function createGroundingContext(options = {}) {
|
|
163
|
+
return {
|
|
164
|
+
tables: [],
|
|
165
|
+
views: [],
|
|
166
|
+
relationships: [],
|
|
167
|
+
info: void 0,
|
|
168
|
+
cache: /* @__PURE__ */ new Map(),
|
|
169
|
+
onProgress: timestampProgress(options.onProgress)
|
|
170
|
+
};
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
// packages/text2sql/src/lib/adapters/adapter.ts
|
|
174
|
+
var Adapter = class {
|
|
175
|
+
#policyAnalyzer;
|
|
176
|
+
/**
|
|
177
|
+
* Injects the strategy that guards validate() and execute().
|
|
178
|
+
*/
|
|
179
|
+
constructor(policyAnalyzer) {
|
|
180
|
+
this.#policyAnalyzer = policyAnalyzer;
|
|
181
|
+
}
|
|
182
|
+
/**
|
|
183
|
+
* Introspect the database schema and return context fragments.
|
|
184
|
+
*
|
|
185
|
+
* Executes all configured groundings to populate the context, then
|
|
186
|
+
* generates fragments from the complete context data.
|
|
187
|
+
*
|
|
188
|
+
* @param ctx - Optional grounding context for sharing state between groundings
|
|
189
|
+
* @returns Array of context fragments representing the database schema
|
|
190
|
+
*/
|
|
191
|
+
async introspect(ctx = createGroundingContext()) {
|
|
192
|
+
for (const fn of this.grounding) {
|
|
193
|
+
const grounding = fn(this);
|
|
194
|
+
if (grounding.phase) {
|
|
195
|
+
ctx.onProgress({
|
|
196
|
+
type: "phase:start",
|
|
197
|
+
phase: grounding.phase,
|
|
198
|
+
message: `Starting ${grounding.name} introspection...`
|
|
199
|
+
});
|
|
200
|
+
}
|
|
201
|
+
try {
|
|
202
|
+
await grounding.execute(ctx);
|
|
203
|
+
} finally {
|
|
204
|
+
if (grounding.phase) {
|
|
205
|
+
ctx.onProgress({
|
|
206
|
+
type: "phase:end",
|
|
207
|
+
phase: grounding.phase,
|
|
208
|
+
message: `Finished ${grounding.name} introspection.`
|
|
209
|
+
});
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
return this.#toSchemaFragments(ctx);
|
|
214
|
+
}
|
|
215
|
+
/**
|
|
216
|
+
* Resolve the allowed entity names (tables + views) from grounding config.
|
|
217
|
+
* Asks each grounding to contribute its entities; annotators contribute
|
|
218
|
+
* nothing, so their metadata scans are skipped.
|
|
219
|
+
* Results are NOT cached — call once and store the result.
|
|
220
|
+
*/
|
|
221
|
+
async resolveAllowedEntities() {
|
|
222
|
+
const ctx = createGroundingContext();
|
|
223
|
+
for (const fn of this.grounding) {
|
|
224
|
+
await fn(this).contributeEntities(ctx);
|
|
225
|
+
}
|
|
226
|
+
return [...ctx.tables.map((t) => t.name), ...ctx.views.map((v) => v.name)];
|
|
227
|
+
}
|
|
228
|
+
/**
|
|
229
|
+
* Convert complete grounding context to schema fragments.
|
|
230
|
+
* Called after all groundings have populated ctx with data.
|
|
231
|
+
*/
|
|
232
|
+
#toSchemaFragments(ctx) {
|
|
233
|
+
const fragments = [];
|
|
234
|
+
if (ctx.info) {
|
|
235
|
+
fragments.push(
|
|
236
|
+
dialectInfo({
|
|
237
|
+
dialect: ctx.info.dialect,
|
|
238
|
+
version: ctx.info.version,
|
|
239
|
+
database: ctx.info.database,
|
|
240
|
+
details: ctx.info.details
|
|
241
|
+
})
|
|
242
|
+
);
|
|
243
|
+
}
|
|
244
|
+
for (const t of ctx.tables) {
|
|
245
|
+
fragments.push(this.#tableToFragment(t));
|
|
246
|
+
}
|
|
247
|
+
for (const v of ctx.views) {
|
|
248
|
+
fragments.push(this.#viewToFragment(v));
|
|
249
|
+
}
|
|
250
|
+
const tableMap = new Map(ctx.tables.map((t) => [t.name, t]));
|
|
251
|
+
const tableColumnSets = new Map(
|
|
252
|
+
ctx.tables.map((t) => [t.name, new Set(t.columns.map((c) => c.name))])
|
|
253
|
+
);
|
|
254
|
+
for (const rel of ctx.relationships) {
|
|
255
|
+
const sourceColumns = tableColumnSets.get(rel.table);
|
|
256
|
+
const targetColumns = tableColumnSets.get(rel.referenced_table);
|
|
257
|
+
if (sourceColumns && rel.from.some((column2) => !sourceColumns.has(column2))) {
|
|
258
|
+
continue;
|
|
259
|
+
}
|
|
260
|
+
if (targetColumns && rel.to.some((column2) => !targetColumns.has(column2))) {
|
|
261
|
+
continue;
|
|
262
|
+
}
|
|
263
|
+
const sourceTable = tableMap.get(rel.table);
|
|
264
|
+
const targetTable = tableMap.get(rel.referenced_table);
|
|
265
|
+
fragments.push(
|
|
266
|
+
this.#relationshipToFragment(rel, sourceTable, targetTable)
|
|
267
|
+
);
|
|
268
|
+
}
|
|
269
|
+
return fragments;
|
|
270
|
+
}
|
|
271
|
+
/**
|
|
272
|
+
* Convert a Table to a table fragment with nested column, index, and constraint fragments.
|
|
273
|
+
*/
|
|
274
|
+
#tableToFragment(t) {
|
|
275
|
+
const pkConstraint = t.constraints?.find((c) => c.type === "PRIMARY_KEY");
|
|
276
|
+
const pkColumns = new Set(pkConstraint?.columns ?? []);
|
|
277
|
+
const notNullColumns = new Set(
|
|
278
|
+
t.constraints?.filter((c) => c.type === "NOT_NULL").flatMap((c) => c.columns ?? []) ?? []
|
|
279
|
+
);
|
|
280
|
+
const defaultByColumn = /* @__PURE__ */ new Map();
|
|
281
|
+
for (const c of t.constraints?.filter((c2) => c2.type === "DEFAULT") ?? []) {
|
|
282
|
+
for (const col of c.columns ?? []) {
|
|
283
|
+
if (c.defaultValue != null) {
|
|
284
|
+
defaultByColumn.set(col, c.defaultValue);
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
const uniqueColumns = new Set(
|
|
289
|
+
t.constraints?.filter((c) => c.type === "UNIQUE" && c.columns?.length === 1).flatMap((c) => c.columns ?? []) ?? []
|
|
290
|
+
);
|
|
291
|
+
const fkByColumn = /* @__PURE__ */ new Map();
|
|
292
|
+
for (const c of t.constraints?.filter((c2) => c2.type === "FOREIGN_KEY") ?? []) {
|
|
293
|
+
const cols = c.columns ?? [];
|
|
294
|
+
const refCols = c.referencedColumns ?? [];
|
|
295
|
+
for (let i = 0; i < cols.length; i++) {
|
|
296
|
+
const refCol = refCols[i] ?? refCols[0] ?? cols[i];
|
|
297
|
+
fkByColumn.set(cols[i], `${c.referencedTable}.${refCol}`);
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
const columnFragments = t.columns.map(
|
|
301
|
+
(col) => column({
|
|
302
|
+
name: col.name,
|
|
303
|
+
type: col.type,
|
|
304
|
+
pk: pkColumns.has(col.name) || void 0,
|
|
305
|
+
fk: fkByColumn.get(col.name),
|
|
306
|
+
unique: uniqueColumns.has(col.name) || void 0,
|
|
307
|
+
notNull: notNullColumns.has(col.name) || void 0,
|
|
308
|
+
default: defaultByColumn.get(col.name),
|
|
309
|
+
indexed: col.isIndexed || void 0,
|
|
310
|
+
values: col.values,
|
|
311
|
+
stats: col.stats
|
|
312
|
+
})
|
|
313
|
+
);
|
|
314
|
+
const presentColumns = new Set(t.columns.map((c) => c.name));
|
|
315
|
+
const indexFragments = (t.indexes ?? []).filter(
|
|
316
|
+
(idx) => idx.columns.every((column2) => presentColumns.has(column2))
|
|
317
|
+
).map(
|
|
318
|
+
(idx) => index({
|
|
319
|
+
name: idx.name,
|
|
320
|
+
columns: idx.columns,
|
|
321
|
+
unique: idx.unique,
|
|
322
|
+
type: idx.type
|
|
323
|
+
})
|
|
324
|
+
);
|
|
325
|
+
const constraintFragments = (t.constraints ?? []).filter(
|
|
326
|
+
(c) => c.type === "CHECK" || c.type === "UNIQUE" && (c.columns?.length ?? 0) > 1
|
|
327
|
+
).filter(
|
|
328
|
+
(c) => !c.columns?.length || c.columns.every((column2) => presentColumns.has(column2))
|
|
329
|
+
).map(
|
|
330
|
+
(c) => constraint({
|
|
331
|
+
name: c.name,
|
|
332
|
+
type: c.type,
|
|
333
|
+
columns: c.columns,
|
|
334
|
+
definition: c.definition
|
|
335
|
+
})
|
|
336
|
+
);
|
|
337
|
+
return table({
|
|
338
|
+
name: t.name,
|
|
339
|
+
schema: t.schema,
|
|
340
|
+
rowCount: t.rowCount,
|
|
341
|
+
sizeHint: t.sizeHint,
|
|
342
|
+
columns: columnFragments,
|
|
343
|
+
indexes: indexFragments.length > 0 ? indexFragments : void 0,
|
|
344
|
+
constraints: constraintFragments.length > 0 ? constraintFragments : void 0
|
|
345
|
+
});
|
|
346
|
+
}
|
|
347
|
+
/**
|
|
348
|
+
* Convert a View to a view fragment with nested column fragments.
|
|
349
|
+
*/
|
|
350
|
+
#viewToFragment(v) {
|
|
351
|
+
const columnFragments = v.columns.map(
|
|
352
|
+
(col) => column({
|
|
353
|
+
name: col.name,
|
|
354
|
+
type: col.type,
|
|
355
|
+
values: col.values,
|
|
356
|
+
stats: col.stats
|
|
357
|
+
})
|
|
358
|
+
);
|
|
359
|
+
return view({
|
|
360
|
+
name: v.name,
|
|
361
|
+
schema: v.schema,
|
|
362
|
+
columns: columnFragments,
|
|
363
|
+
definition: v.definition
|
|
364
|
+
});
|
|
365
|
+
}
|
|
366
|
+
/**
|
|
367
|
+
* Convert a Relationship to a relationship fragment.
|
|
368
|
+
* Infers cardinality from row counts if available.
|
|
369
|
+
*/
|
|
370
|
+
#relationshipToFragment(rel, sourceTable, targetTable) {
|
|
371
|
+
const sourceCount = sourceTable?.rowCount;
|
|
372
|
+
const targetCount = targetTable?.rowCount;
|
|
373
|
+
let cardinality;
|
|
374
|
+
if (sourceCount != null && targetCount != null && targetCount > 0) {
|
|
375
|
+
const ratio = sourceCount / targetCount;
|
|
376
|
+
if (ratio > 5) {
|
|
377
|
+
cardinality = "many-to-one";
|
|
378
|
+
} else if (ratio < 1.2 && ratio > 0.8) {
|
|
379
|
+
cardinality = "one-to-one";
|
|
380
|
+
} else if (ratio < 0.2) {
|
|
381
|
+
cardinality = "one-to-many";
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
return relationship({
|
|
385
|
+
from: { table: rel.table, columns: rel.from },
|
|
386
|
+
to: { table: rel.referenced_table, columns: rel.to },
|
|
387
|
+
cardinality
|
|
388
|
+
});
|
|
389
|
+
}
|
|
390
|
+
format(sql) {
|
|
391
|
+
const decoded = this.#decodeShellEscapes(sql);
|
|
392
|
+
try {
|
|
393
|
+
return formatSql(decoded, { language: this.formatterLanguage });
|
|
394
|
+
} catch {
|
|
395
|
+
return decoded;
|
|
396
|
+
}
|
|
397
|
+
}
|
|
398
|
+
#decodeShellEscapes(sql) {
|
|
399
|
+
return sql.replace(/\\n/g, "\n").replace(/\\t/g, " ").replace(/\\([().*])/g, "$1").replace(/\\(?=$)/gm, "");
|
|
400
|
+
}
|
|
401
|
+
#cachedAllowedEntities = null;
|
|
402
|
+
async #resolveScope() {
|
|
403
|
+
if (this.#cachedAllowedEntities) return this.#cachedAllowedEntities;
|
|
404
|
+
this.#cachedAllowedEntities = await this.resolveAllowedEntities();
|
|
405
|
+
return this.#cachedAllowedEntities;
|
|
406
|
+
}
|
|
407
|
+
async #analyzePolicy(sql) {
|
|
408
|
+
return this.#policyAnalyzer.analyze(sql, {
|
|
409
|
+
resolveAllowedEntities: () => this.#resolveScope()
|
|
410
|
+
});
|
|
411
|
+
}
|
|
412
|
+
async validate(sql) {
|
|
413
|
+
const decoded = this.#decodeShellEscapes(sql);
|
|
414
|
+
const violation = await this.#analyzePolicy(decoded);
|
|
415
|
+
if (violation?.kind === "read-only") return violation.message;
|
|
416
|
+
if (violation?.kind === "scope") return JSON.stringify(violation.payload);
|
|
417
|
+
return this.validateImpl(decoded);
|
|
418
|
+
}
|
|
419
|
+
async execute(sql) {
|
|
420
|
+
const decoded = this.#decodeShellEscapes(sql);
|
|
421
|
+
const violation = await this.#analyzePolicy(decoded);
|
|
422
|
+
if (violation?.kind === "read-only") {
|
|
423
|
+
throw new SQLReadOnlyError(violation.message);
|
|
424
|
+
}
|
|
425
|
+
if (violation?.kind === "scope") {
|
|
426
|
+
throw new SQLScopeError(violation.payload);
|
|
427
|
+
}
|
|
428
|
+
return this.executeImpl(decoded);
|
|
429
|
+
}
|
|
430
|
+
/**
|
|
431
|
+
* Convert unknown database value to number.
|
|
432
|
+
* Handles number, bigint, and string types.
|
|
433
|
+
*/
|
|
434
|
+
toNumber(value) {
|
|
435
|
+
if (typeof value === "number" && Number.isFinite(value)) {
|
|
436
|
+
return value;
|
|
437
|
+
}
|
|
438
|
+
if (typeof value === "bigint") {
|
|
439
|
+
return Number(value);
|
|
440
|
+
}
|
|
441
|
+
if (typeof value === "string" && value.trim() !== "") {
|
|
442
|
+
const parsed = Number(value);
|
|
443
|
+
return Number.isFinite(parsed) ? parsed : void 0;
|
|
444
|
+
}
|
|
445
|
+
return void 0;
|
|
446
|
+
}
|
|
447
|
+
/**
|
|
448
|
+
* Parse a potentially qualified table name into schema and table parts.
|
|
449
|
+
*/
|
|
450
|
+
parseTableName(name) {
|
|
451
|
+
if (name.includes(".")) {
|
|
452
|
+
const [schema, ...rest] = name.split(".");
|
|
453
|
+
return { schema, table: rest.join(".") };
|
|
454
|
+
}
|
|
455
|
+
return { schema: this.defaultSchema ?? "", table: name };
|
|
456
|
+
}
|
|
457
|
+
/**
|
|
458
|
+
* Escape a string value for use in SQL string literals (single quotes).
|
|
459
|
+
* Used in WHERE clauses like: WHERE name = '${escapeString(value)}'
|
|
460
|
+
*/
|
|
461
|
+
escapeString(value) {
|
|
462
|
+
return value.replace(/'/g, "''");
|
|
463
|
+
}
|
|
464
|
+
/**
|
|
465
|
+
* Build a SQL filter clause to include/exclude schemas.
|
|
466
|
+
* @param columnName - The schema column name (e.g., 'TABLE_SCHEMA')
|
|
467
|
+
* @param allowedSchemas - If provided, filter to these schemas only
|
|
468
|
+
*/
|
|
469
|
+
buildSchemaFilter(columnName, allowedSchemas) {
|
|
470
|
+
if (allowedSchemas && allowedSchemas.length > 0) {
|
|
471
|
+
const values = allowedSchemas.map((s) => `'${this.escapeString(s)}'`).join(", ");
|
|
472
|
+
return `AND ${columnName} IN (${values})`;
|
|
473
|
+
}
|
|
474
|
+
if (this.systemSchemas.length > 0) {
|
|
475
|
+
const values = this.systemSchemas.map((s) => `'${this.escapeString(s)}'`).join(", ");
|
|
476
|
+
return `AND ${columnName} NOT IN (${values})`;
|
|
477
|
+
}
|
|
478
|
+
return "";
|
|
479
|
+
}
|
|
480
|
+
};
|
|
481
|
+
|
|
482
|
+
// packages/text2sql/src/lib/sql-scope-error.ts
|
|
483
|
+
function buildScopeParseErrorPayload(sql, parserDialect, error) {
|
|
484
|
+
const parserError = error instanceof Error ? error.message : String(error ?? "Unknown error");
|
|
485
|
+
return {
|
|
486
|
+
error: `SQL scope analysis failed before validation/execution: ${parserError}`,
|
|
487
|
+
error_type: "SQL_SCOPE_PARSE_ERROR",
|
|
488
|
+
suggestion: "Rewrite the query into simpler SQL that can be analyzed safely, or extend parser coverage for this dialect feature.",
|
|
489
|
+
sql_attempted: sql,
|
|
490
|
+
parser_dialect: parserDialect,
|
|
491
|
+
parser_error: parserError
|
|
492
|
+
};
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
// packages/text2sql/src/lib/adapters/clickhouse/clickhouse.sql-policy.ts
|
|
496
|
+
var READ_ONLY_MESSAGE = "only SELECT queries allowed";
|
|
497
|
+
var ClickHouseSqlPolicyAnalyzer = class {
|
|
498
|
+
#query;
|
|
499
|
+
#readonlyCheck;
|
|
500
|
+
#currentDatabase;
|
|
501
|
+
constructor(query) {
|
|
502
|
+
this.#query = query;
|
|
503
|
+
}
|
|
504
|
+
async analyze(sql, context) {
|
|
505
|
+
try {
|
|
506
|
+
await (this.#readonlyCheck ??= this.#verifyReadonly());
|
|
507
|
+
const ast = await this.#explain(`EXPLAIN AST ${sql}`);
|
|
508
|
+
const astAnalysis = analyzeAst(ast);
|
|
509
|
+
if (!astAnalysis.isSelect) {
|
|
510
|
+
return { kind: "read-only", message: READ_ONLY_MESSAGE };
|
|
511
|
+
}
|
|
512
|
+
if (await this.#hasUnsafeFunction(astAnalysis.functions)) {
|
|
513
|
+
return { kind: "read-only", message: READ_ONLY_MESSAGE };
|
|
514
|
+
}
|
|
515
|
+
const syntacticTree = await this.#explain(
|
|
516
|
+
`EXPLAIN QUERY TREE run_passes = 0 ${sql}`
|
|
517
|
+
);
|
|
518
|
+
const analyzedTree = await this.#explain(`EXPLAIN QUERY TREE ${sql}`);
|
|
519
|
+
const syntacticRelations = readQueryTreeRelations(
|
|
520
|
+
syntacticTree,
|
|
521
|
+
"syntactic"
|
|
522
|
+
);
|
|
523
|
+
const analyzedRelations = readQueryTreeRelations(
|
|
524
|
+
analyzedTree,
|
|
525
|
+
"analyzed"
|
|
526
|
+
);
|
|
527
|
+
if (!this.#currentDatabase) {
|
|
528
|
+
throw new Error("ClickHouse current database was not established.");
|
|
529
|
+
}
|
|
530
|
+
const relations = [
|
|
531
|
+
...new Set(
|
|
532
|
+
[...syntacticRelations, ...analyzedRelations].map(
|
|
533
|
+
(relation) => qualifyRelation(relation, this.#currentDatabase)
|
|
534
|
+
)
|
|
535
|
+
)
|
|
536
|
+
];
|
|
537
|
+
const allowed = await context.resolveAllowedEntities();
|
|
538
|
+
const rejected = relations.filter(
|
|
539
|
+
(relation) => !isAllowedRelation(relation, allowed)
|
|
540
|
+
);
|
|
541
|
+
return rejected.length ? {
|
|
542
|
+
kind: "scope",
|
|
543
|
+
payload: buildOutOfScopePayload(sql, rejected, [...allowed])
|
|
544
|
+
} : null;
|
|
545
|
+
} catch (error) {
|
|
546
|
+
return {
|
|
547
|
+
kind: "scope",
|
|
548
|
+
payload: buildScopeParseErrorPayload(sql, "clickhouse", error)
|
|
549
|
+
};
|
|
550
|
+
}
|
|
551
|
+
}
|
|
552
|
+
async #verifyReadonly() {
|
|
553
|
+
const rows = readRows(
|
|
554
|
+
await this.#query(
|
|
555
|
+
"SELECT getSetting('readonly') AS readonly, currentDatabase() AS database"
|
|
556
|
+
)
|
|
557
|
+
);
|
|
558
|
+
const row = rows[0];
|
|
559
|
+
if (rows.length !== 1 || !row || Object.keys(row).length !== 2 || typeof row.database !== "string" || row.database.length === 0) {
|
|
560
|
+
throw new Error(
|
|
561
|
+
"ClickHouse readonly probe returned an unknown row shape."
|
|
562
|
+
);
|
|
563
|
+
}
|
|
564
|
+
const value = row.readonly;
|
|
565
|
+
if (value !== 1 && value !== "1") {
|
|
566
|
+
throw new Error(
|
|
567
|
+
`ClickHouse connection must have effective readonly = 1; received ${String(value)}.`
|
|
568
|
+
);
|
|
569
|
+
}
|
|
570
|
+
this.#currentDatabase = row.database;
|
|
571
|
+
}
|
|
572
|
+
async #explain(sql) {
|
|
573
|
+
const rows = readRows(await this.#query(sql));
|
|
574
|
+
if (rows.length === 0) {
|
|
575
|
+
throw new Error("ClickHouse EXPLAIN returned no rows.");
|
|
576
|
+
}
|
|
577
|
+
return rows.map((row) => {
|
|
578
|
+
if (Object.keys(row).length !== 1 || typeof row.explain !== "string") {
|
|
579
|
+
throw new Error(
|
|
580
|
+
"ClickHouse EXPLAIN must return rows with exactly one string `explain` column."
|
|
581
|
+
);
|
|
582
|
+
}
|
|
583
|
+
return row.explain;
|
|
584
|
+
});
|
|
585
|
+
}
|
|
586
|
+
async #hasUnsafeFunction(functions) {
|
|
587
|
+
if (functions.some(isScopeBypassingFunction)) return true;
|
|
588
|
+
if (functions.length === 0) return false;
|
|
589
|
+
const names = [...new Set(functions)].map((name) => `'${name.replaceAll("'", "''")}'`).join(", ");
|
|
590
|
+
const rows = readRows(
|
|
591
|
+
await this.#query(`
|
|
592
|
+
SELECT name, toString(origin) AS origin
|
|
593
|
+
FROM system.functions
|
|
594
|
+
WHERE name IN (${names})
|
|
595
|
+
AND origin != 'System'
|
|
596
|
+
`)
|
|
597
|
+
);
|
|
598
|
+
for (const row of rows) {
|
|
599
|
+
if (typeof row.name !== "string" || typeof row.origin !== "string") {
|
|
600
|
+
throw new Error(
|
|
601
|
+
"ClickHouse system.functions returned an unknown row shape."
|
|
602
|
+
);
|
|
603
|
+
}
|
|
604
|
+
if (row.origin !== "SQLUserDefined" && row.origin !== "ExecutableUserDefined" && row.origin !== "WasmUserDefined") {
|
|
605
|
+
throw new Error(`Unknown ClickHouse function origin: ${row.origin}`);
|
|
606
|
+
}
|
|
607
|
+
}
|
|
608
|
+
return rows.length > 0;
|
|
609
|
+
}
|
|
610
|
+
};
|
|
611
|
+
function isScopeBypassingFunction(name) {
|
|
612
|
+
const normalized = name.toLocaleLowerCase("en-US");
|
|
613
|
+
return normalized.startsWith("dict") || normalized.startsWith("joinget");
|
|
614
|
+
}
|
|
615
|
+
function readRows(result) {
|
|
616
|
+
const rows = Array.isArray(result) ? result : result && typeof result === "object" && "data" in result ? result.data : result && typeof result === "object" && "rows" in result ? result.rows : void 0;
|
|
617
|
+
if (!Array.isArray(rows) || rows.some((row) => !isRecord(row))) {
|
|
618
|
+
throw new Error(
|
|
619
|
+
"ClickHouse query callback must return row objects as an array, { data }, or { rows }."
|
|
620
|
+
);
|
|
621
|
+
}
|
|
622
|
+
return rows;
|
|
623
|
+
}
|
|
624
|
+
function analyzeAst(lines) {
|
|
625
|
+
const rootLines = lines.filter(
|
|
626
|
+
(line) => line.length - line.trimStart().length === 0
|
|
627
|
+
);
|
|
628
|
+
if (rootLines.length !== 1 || !rootLines[0]?.startsWith("SelectWithUnionQuery")) {
|
|
629
|
+
return { isSelect: false, functions: [] };
|
|
630
|
+
}
|
|
631
|
+
const roots = parseIndentedTree(lines, validateAstLine);
|
|
632
|
+
const root = roots[0];
|
|
633
|
+
const isSelect = roots.length === 1 && !!root && !root.children.some((child) => child.text.startsWith("Literal ")) && !containsTableFunction(root);
|
|
634
|
+
return {
|
|
635
|
+
isSelect,
|
|
636
|
+
functions: isSelect && root ? collectAstFunctions(root) : []
|
|
637
|
+
};
|
|
638
|
+
}
|
|
639
|
+
function validateAstLine(text) {
|
|
640
|
+
const type = /^([A-Za-z][A-Za-z0-9]*)/.exec(text)?.[1];
|
|
641
|
+
const allowed = /* @__PURE__ */ new Set([
|
|
642
|
+
"ArrayJoin",
|
|
643
|
+
"Asterisk",
|
|
644
|
+
"ExpressionList",
|
|
645
|
+
"Function",
|
|
646
|
+
"Identifier",
|
|
647
|
+
"Literal",
|
|
648
|
+
"OrderByElement",
|
|
649
|
+
"SelectQuery",
|
|
650
|
+
"SelectWithUnionQuery",
|
|
651
|
+
"Subquery",
|
|
652
|
+
"TableExpression",
|
|
653
|
+
"TableIdentifier",
|
|
654
|
+
"TableJoin",
|
|
655
|
+
"TablesInSelectQuery",
|
|
656
|
+
"TablesInSelectQueryElement",
|
|
657
|
+
"WithElement",
|
|
658
|
+
"WindowDefinition"
|
|
659
|
+
]);
|
|
660
|
+
if (!type || !allowed.has(type)) {
|
|
661
|
+
throw new Error(`Unknown ClickHouse AST node: ${text}`);
|
|
662
|
+
}
|
|
663
|
+
}
|
|
664
|
+
function containsTableFunction(node) {
|
|
665
|
+
if (node.text.startsWith("TableExpression") && node.children.some((child) => child.text.startsWith("Function "))) {
|
|
666
|
+
return true;
|
|
667
|
+
}
|
|
668
|
+
return node.children.some(containsTableFunction);
|
|
669
|
+
}
|
|
670
|
+
function collectAstFunctions(node) {
|
|
671
|
+
const functions = [];
|
|
672
|
+
if (node.text.startsWith("Function ")) {
|
|
673
|
+
const name = /^Function (\S+)/.exec(node.text)?.[1];
|
|
674
|
+
if (!name) throw new Error(`Unknown ClickHouse AST function: ${node.text}`);
|
|
675
|
+
functions.push(name);
|
|
676
|
+
}
|
|
677
|
+
for (const child of node.children)
|
|
678
|
+
functions.push(...collectAstFunctions(child));
|
|
679
|
+
return functions;
|
|
680
|
+
}
|
|
681
|
+
function readQueryTreeRelations(lines, phase) {
|
|
682
|
+
const roots = parseIndentedTree(
|
|
683
|
+
lines,
|
|
684
|
+
(text, parent) => validateQueryTreeLine(text, parent)
|
|
685
|
+
);
|
|
686
|
+
if (roots.length !== 1 || !roots[0]?.text.startsWith("QUERY ") && !roots[0]?.text.startsWith("UNION ")) {
|
|
687
|
+
throw new Error(`Unknown ClickHouse ${phase} query-tree root.`);
|
|
688
|
+
}
|
|
689
|
+
const relations = [];
|
|
690
|
+
const cteNames = /* @__PURE__ */ new Set();
|
|
691
|
+
collectCteNames(roots[0], cteNames);
|
|
692
|
+
visitQueryTree(roots[0], false, phase, cteNames, relations);
|
|
693
|
+
return relations;
|
|
694
|
+
}
|
|
695
|
+
function validateQueryTreeLine(text, parent) {
|
|
696
|
+
const nodePatterns = [
|
|
697
|
+
/^QUERY id:/,
|
|
698
|
+
/^UNION id:/,
|
|
699
|
+
/^QUERIES$/,
|
|
700
|
+
/^PROJECTION(?: COLUMNS)?$/,
|
|
701
|
+
/^WITH$/,
|
|
702
|
+
/^LIST id:/,
|
|
703
|
+
/^MATCHER id:/,
|
|
704
|
+
/^COLUMN id:/,
|
|
705
|
+
/^JOIN TREE$/,
|
|
706
|
+
/^ARRAY_JOIN id:/,
|
|
707
|
+
/^IDENTIFIER id:/,
|
|
708
|
+
/^TABLE id:/,
|
|
709
|
+
/^JOIN id:/,
|
|
710
|
+
/^(?:(?:LEFT|RIGHT) )?TABLE EXPRESSION$/,
|
|
711
|
+
/^JOIN EXPRESSIONS?$/,
|
|
712
|
+
/^FUNCTION id:/,
|
|
713
|
+
/^ARGUMENTS$/,
|
|
714
|
+
/^CONSTANT id:/,
|
|
715
|
+
/^PREWHERE$/,
|
|
716
|
+
/^WHERE$/,
|
|
717
|
+
/^WINDOW$/,
|
|
718
|
+
/^WINDOW id:/,
|
|
719
|
+
/^PARTITION BY$/,
|
|
720
|
+
/^QUALIFY$/,
|
|
721
|
+
/^GROUP BY$/,
|
|
722
|
+
/^HAVING$/,
|
|
723
|
+
/^ORDER BY$/,
|
|
724
|
+
/^SORT id:/,
|
|
725
|
+
/^EXPRESSION(?: .*)?$/,
|
|
726
|
+
/^LIMIT BY LIMIT$/,
|
|
727
|
+
/^LIMIT BY$/,
|
|
728
|
+
/^LIMIT$/,
|
|
729
|
+
/^OFFSET$/
|
|
730
|
+
];
|
|
731
|
+
if (nodePatterns.some((pattern) => pattern.test(text))) return;
|
|
732
|
+
if (parent?.text === "PROJECTION COLUMNS" && /^\S+\s+\S+/.test(text)) {
|
|
733
|
+
return;
|
|
734
|
+
}
|
|
735
|
+
throw new Error(`Unknown ClickHouse query-tree node: ${text}`);
|
|
736
|
+
}
|
|
737
|
+
function visitQueryTree(node, relationPosition, phase, cteNames, relations) {
|
|
738
|
+
if (relationPosition && node.text.startsWith("IDENTIFIER ")) {
|
|
739
|
+
const identifier = /(?:^|, )identifier: (.+)$/.exec(node.text)?.[1];
|
|
740
|
+
if (!identifier) {
|
|
741
|
+
throw new Error(`Unknown ClickHouse ${phase} relation: ${node.text}`);
|
|
742
|
+
}
|
|
743
|
+
if (!cteNames.has(identifier)) {
|
|
744
|
+
relations.push(identifier);
|
|
745
|
+
}
|
|
746
|
+
} else if (relationPosition && node.text.startsWith("TABLE ")) {
|
|
747
|
+
const table2 = /(?:^|, )table_name: ([^,]+)(?:,|$)/.exec(node.text)?.[1];
|
|
748
|
+
if (!table2) {
|
|
749
|
+
throw new Error(`Unknown ClickHouse ${phase} relation: ${node.text}`);
|
|
750
|
+
}
|
|
751
|
+
relations.push(table2);
|
|
752
|
+
}
|
|
753
|
+
for (const child of node.children) {
|
|
754
|
+
const childrenAreRelations = node.text === "JOIN TREE" || node.text === "LEFT TABLE EXPRESSION" || node.text === "RIGHT TABLE EXPRESSION" || node.text === "TABLE EXPRESSION";
|
|
755
|
+
visitQueryTree(child, childrenAreRelations, phase, cteNames, relations);
|
|
756
|
+
}
|
|
757
|
+
}
|
|
758
|
+
function collectCteNames(node, cteNames) {
|
|
759
|
+
const name = /(?:^|, )cte_name: ([^,]+)(?:,|$)/.exec(node.text)?.[1];
|
|
760
|
+
if (name) cteNames.add(name);
|
|
761
|
+
for (const child of node.children) collectCteNames(child, cteNames);
|
|
762
|
+
}
|
|
763
|
+
function parseIndentedTree(lines, validate) {
|
|
764
|
+
const roots = [];
|
|
765
|
+
const stack = [];
|
|
766
|
+
for (const line of lines) {
|
|
767
|
+
if (!line.trim())
|
|
768
|
+
throw new Error("ClickHouse EXPLAIN returned an empty line.");
|
|
769
|
+
const indent = line.length - line.trimStart().length;
|
|
770
|
+
const text = line.trimStart();
|
|
771
|
+
while (stack.length && stack.at(-1).indent >= indent) stack.pop();
|
|
772
|
+
const parent = stack.at(-1)?.node;
|
|
773
|
+
validate(text, parent);
|
|
774
|
+
const node = { text, children: [] };
|
|
775
|
+
if (parent) parent.children.push(node);
|
|
776
|
+
else roots.push(node);
|
|
777
|
+
stack.push({ indent, node });
|
|
778
|
+
}
|
|
779
|
+
return roots;
|
|
780
|
+
}
|
|
781
|
+
function isAllowedRelation(relation, allowedEntities) {
|
|
782
|
+
return allowedEntities.includes(relation);
|
|
783
|
+
}
|
|
784
|
+
function qualifyRelation(relation, database) {
|
|
785
|
+
return relation.includes(".") ? relation : `${database}.${relation}`;
|
|
786
|
+
}
|
|
787
|
+
function buildOutOfScopePayload(sql, referencedEntities, allowedEntities) {
|
|
788
|
+
return {
|
|
789
|
+
error: `Query references entities outside grounded scope: ${referencedEntities.join(", ")}`,
|
|
790
|
+
error_type: "OUT_OF_SCOPE",
|
|
791
|
+
suggestion: "Restrict the query to grounded tables/views or expand grounding to include the referenced entities.",
|
|
792
|
+
sql_attempted: sql,
|
|
793
|
+
referenced_entities: referencedEntities,
|
|
794
|
+
allowed_entities: allowedEntities
|
|
795
|
+
};
|
|
796
|
+
}
|
|
797
|
+
function isRecord(value) {
|
|
798
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
799
|
+
}
|
|
800
|
+
|
|
801
|
+
// packages/text2sql/src/lib/adapters/clickhouse/clickhouse.ts
|
|
802
|
+
var ClickHouse = class extends Adapter {
|
|
803
|
+
#options;
|
|
804
|
+
grounding;
|
|
805
|
+
defaultSchema;
|
|
806
|
+
systemSchemas = [
|
|
807
|
+
"system",
|
|
808
|
+
"information_schema",
|
|
809
|
+
"INFORMATION_SCHEMA"
|
|
810
|
+
];
|
|
811
|
+
formatterLanguage = "clickhouse";
|
|
812
|
+
constructor(options) {
|
|
813
|
+
if (!options || typeof options.execute !== "function") {
|
|
814
|
+
throw new Error("ClickHouse adapter requires an execute(sql) function.");
|
|
815
|
+
}
|
|
816
|
+
if (typeof options.validate !== "function") {
|
|
817
|
+
throw new Error("ClickHouse adapter requires a validate(sql) function.");
|
|
818
|
+
}
|
|
819
|
+
super(new ClickHouseSqlPolicyAnalyzer(options.execute));
|
|
820
|
+
this.#options = options;
|
|
821
|
+
this.grounding = options.grounding ?? [];
|
|
822
|
+
this.defaultSchema = options.defaultDatabase;
|
|
823
|
+
}
|
|
824
|
+
async executeImpl(sql) {
|
|
825
|
+
return rowsFromResult(await this.#options.execute(sql));
|
|
826
|
+
}
|
|
827
|
+
async validateImpl(sql) {
|
|
828
|
+
try {
|
|
829
|
+
return await this.#options.validate(sql);
|
|
830
|
+
} catch (error) {
|
|
831
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
832
|
+
return JSON.stringify({
|
|
833
|
+
error: message,
|
|
834
|
+
error_type: "CLICKHOUSE_ERROR",
|
|
835
|
+
suggestion: "Review the ClickHouse syntax and verify referenced databases, tables, columns, and functions.",
|
|
836
|
+
sql_attempted: sql
|
|
837
|
+
});
|
|
838
|
+
}
|
|
839
|
+
}
|
|
840
|
+
async runQuery(sql) {
|
|
841
|
+
return rowsFromResult(await this.#options.execute(sql));
|
|
842
|
+
}
|
|
843
|
+
quoteIdentifier(name) {
|
|
844
|
+
return `\`${name.replace(/`/g, "``")}\``;
|
|
845
|
+
}
|
|
846
|
+
escape(value) {
|
|
847
|
+
return value.replace(/`/g, "``");
|
|
848
|
+
}
|
|
849
|
+
buildSampleRowsQuery(tableName, columns, limit) {
|
|
850
|
+
const { schema, table: table2 } = this.parseTableName(tableName);
|
|
851
|
+
const relation = schema ? `${this.quoteIdentifier(schema)}.${this.quoteIdentifier(table2)}` : this.quoteIdentifier(table2);
|
|
852
|
+
const projection = columns?.length ? columns.map((column2) => this.quoteIdentifier(column2)).join(", ") : "*";
|
|
853
|
+
return `SELECT ${projection} FROM ${relation} LIMIT ${limit}`;
|
|
854
|
+
}
|
|
855
|
+
};
|
|
856
|
+
function rowsFromResult(result) {
|
|
857
|
+
if (Array.isArray(result)) return result;
|
|
858
|
+
if (result && typeof result === "object") {
|
|
859
|
+
const candidate = result;
|
|
860
|
+
if (Array.isArray(candidate.data)) return candidate.data;
|
|
861
|
+
if (Array.isArray(candidate.rows)) return candidate.rows;
|
|
862
|
+
}
|
|
863
|
+
throw new Error(
|
|
864
|
+
"ClickHouse execute() must return an array of rows, { data: rows }, or { rows }."
|
|
865
|
+
);
|
|
866
|
+
}
|
|
867
|
+
|
|
868
|
+
// packages/text2sql/src/lib/adapters/groundings/abstract.grounding.ts
|
|
869
|
+
function filterColumns(columns, filter) {
|
|
870
|
+
if (Array.isArray(filter)) {
|
|
871
|
+
return columns.filter((col) => filter.includes(col.name));
|
|
872
|
+
}
|
|
873
|
+
if (filter instanceof RegExp) {
|
|
874
|
+
return columns.filter((col) => filter.test(col.name));
|
|
875
|
+
}
|
|
876
|
+
return columns.filter((col) => filter(col.name));
|
|
877
|
+
}
|
|
878
|
+
function applyColumnFilter(entity, columnsConfig) {
|
|
879
|
+
if (!columnsConfig) return entity;
|
|
880
|
+
const filter = columnsConfig[entity.name];
|
|
881
|
+
if (!filter) return entity;
|
|
882
|
+
return { ...entity, columns: filterColumns(entity.columns, filter) };
|
|
883
|
+
}
|
|
884
|
+
var AbstractGrounding = class {
|
|
885
|
+
/**
|
|
886
|
+
* Grounding identifier for debugging/logging.
|
|
887
|
+
*/
|
|
888
|
+
name;
|
|
889
|
+
phase;
|
|
890
|
+
constructor(name, phase) {
|
|
891
|
+
this.name = name;
|
|
892
|
+
this.phase = phase;
|
|
893
|
+
}
|
|
894
|
+
/**
|
|
895
|
+
* Populate ctx with the tables/views this grounding makes available for scope
|
|
896
|
+
* resolution. Annotators decorate entities other groundings produced and add
|
|
897
|
+
* no membership, so the default is a no-op; entity-producing groundings
|
|
898
|
+
* override it. This lets scope resolution skip the annotators' metadata scans.
|
|
899
|
+
*
|
|
900
|
+
* @param _ctx - Shared context for accumulating entity membership
|
|
901
|
+
*/
|
|
902
|
+
async contributeEntities(_ctx) {
|
|
903
|
+
}
|
|
904
|
+
};
|
|
905
|
+
|
|
906
|
+
// packages/text2sql/src/lib/adapters/groundings/constraint.grounding.ts
|
|
907
|
+
var ConstraintGrounding = class extends AbstractGrounding {
|
|
908
|
+
constructor(config = {}) {
|
|
909
|
+
super("constraint", "constraints");
|
|
910
|
+
}
|
|
911
|
+
/**
|
|
912
|
+
* Execute the grounding process.
|
|
913
|
+
* Annotates tables in ctx.tables with their constraints.
|
|
914
|
+
*/
|
|
915
|
+
async execute(ctx) {
|
|
916
|
+
const total = ctx.tables.length;
|
|
917
|
+
for (let i = 0; i < ctx.tables.length; i++) {
|
|
918
|
+
const table2 = ctx.tables[i];
|
|
919
|
+
ctx.onProgress({
|
|
920
|
+
type: "phase:progress",
|
|
921
|
+
phase: "constraints",
|
|
922
|
+
table: table2.name,
|
|
923
|
+
message: `Loading constraints for ${table2.name}...`,
|
|
924
|
+
current: i + 1,
|
|
925
|
+
total
|
|
926
|
+
});
|
|
927
|
+
try {
|
|
928
|
+
table2.constraints = await this.getConstraints(table2.name);
|
|
929
|
+
} catch (error) {
|
|
930
|
+
console.warn("Error collecting constraints for", table2.name, error);
|
|
931
|
+
}
|
|
932
|
+
}
|
|
933
|
+
}
|
|
934
|
+
};
|
|
935
|
+
|
|
936
|
+
// packages/text2sql/src/lib/adapters/clickhouse/constraint.clickhouse.grounding.ts
|
|
937
|
+
var ClickHouseConstraintGrounding = class extends ConstraintGrounding {
|
|
938
|
+
#adapter;
|
|
939
|
+
constructor(adapter, config = {}) {
|
|
940
|
+
super(config);
|
|
941
|
+
this.#adapter = adapter;
|
|
942
|
+
}
|
|
943
|
+
async getConstraints(tableName) {
|
|
944
|
+
const { schema, table: table2 } = this.#adapter.parseTableName(tableName);
|
|
945
|
+
if (!schema) return [];
|
|
946
|
+
const rows = await this.#adapter.runQuery(`
|
|
947
|
+
SELECT name, type, default_kind, default_expression, is_in_primary_key
|
|
948
|
+
FROM system.columns
|
|
949
|
+
WHERE database = '${this.#adapter.escapeString(schema)}'
|
|
950
|
+
AND table = '${this.#adapter.escapeString(table2)}'
|
|
951
|
+
ORDER BY position
|
|
952
|
+
`);
|
|
953
|
+
const constraints2 = [];
|
|
954
|
+
const primaryKey = rows.filter((row) => Number(row.is_in_primary_key) === 1).map((row) => row.name);
|
|
955
|
+
if (primaryKey.length > 0) {
|
|
956
|
+
constraints2.push({
|
|
957
|
+
name: "PRIMARY_KEY",
|
|
958
|
+
type: "PRIMARY_KEY",
|
|
959
|
+
columns: primaryKey
|
|
960
|
+
});
|
|
961
|
+
}
|
|
962
|
+
for (const row of rows) {
|
|
963
|
+
if (!hasNullableType(row.type)) {
|
|
964
|
+
constraints2.push({
|
|
965
|
+
name: `${row.name}_not_null`,
|
|
966
|
+
type: "NOT_NULL",
|
|
967
|
+
columns: [row.name]
|
|
968
|
+
});
|
|
969
|
+
}
|
|
970
|
+
if (row.default_kind === "DEFAULT" && row.default_expression) {
|
|
971
|
+
constraints2.push({
|
|
972
|
+
name: `${row.name}_default`,
|
|
973
|
+
type: "DEFAULT",
|
|
974
|
+
columns: [row.name],
|
|
975
|
+
defaultValue: row.default_expression
|
|
976
|
+
});
|
|
977
|
+
}
|
|
978
|
+
}
|
|
979
|
+
return constraints2;
|
|
980
|
+
}
|
|
981
|
+
};
|
|
982
|
+
function hasNullableType(type) {
|
|
983
|
+
return /(?:^|\()Nullable\(/.test(type);
|
|
984
|
+
}
|
|
985
|
+
|
|
986
|
+
// packages/text2sql/src/lib/adapters/groundings/indexes.grounding.ts
|
|
987
|
+
var IndexesGrounding = class extends AbstractGrounding {
|
|
988
|
+
constructor(config = {}) {
|
|
989
|
+
super("index", "indexes");
|
|
990
|
+
}
|
|
991
|
+
/**
|
|
992
|
+
* Execute the grounding process.
|
|
993
|
+
* Annotates tables in ctx.tables with their indexes and marks indexed columns.
|
|
994
|
+
*/
|
|
995
|
+
async execute(ctx) {
|
|
996
|
+
const total = ctx.tables.length;
|
|
997
|
+
for (let i = 0; i < ctx.tables.length; i++) {
|
|
998
|
+
const table2 = ctx.tables[i];
|
|
999
|
+
ctx.onProgress({
|
|
1000
|
+
type: "phase:progress",
|
|
1001
|
+
phase: "indexes",
|
|
1002
|
+
table: table2.name,
|
|
1003
|
+
message: `Loading indexes for ${table2.name}...`,
|
|
1004
|
+
current: i + 1,
|
|
1005
|
+
total
|
|
1006
|
+
});
|
|
1007
|
+
table2.indexes = await this.getIndexes(table2.name);
|
|
1008
|
+
for (const index2 of table2.indexes ?? []) {
|
|
1009
|
+
for (const colName of index2.columns) {
|
|
1010
|
+
const column2 = table2.columns.find((c) => c.name === colName);
|
|
1011
|
+
if (column2) {
|
|
1012
|
+
column2.isIndexed = true;
|
|
1013
|
+
}
|
|
1014
|
+
}
|
|
1015
|
+
}
|
|
1016
|
+
}
|
|
1017
|
+
}
|
|
1018
|
+
};
|
|
1019
|
+
|
|
1020
|
+
// packages/text2sql/src/lib/adapters/clickhouse/indexes.clickhouse.grounding.ts
|
|
1021
|
+
var ClickHouseIndexesGrounding = class extends IndexesGrounding {
|
|
1022
|
+
#adapter;
|
|
1023
|
+
constructor(adapter, config = {}) {
|
|
1024
|
+
super(config);
|
|
1025
|
+
this.#adapter = adapter;
|
|
1026
|
+
}
|
|
1027
|
+
async getIndexes(tableName) {
|
|
1028
|
+
const { schema, table: table2 } = this.#adapter.parseTableName(tableName);
|
|
1029
|
+
if (!schema) return [];
|
|
1030
|
+
const [primaryKeyRows, indexRows] = await Promise.all([
|
|
1031
|
+
this.#adapter.runQuery(`
|
|
1032
|
+
SELECT name
|
|
1033
|
+
FROM system.columns
|
|
1034
|
+
WHERE database = '${this.#adapter.escapeString(schema)}'
|
|
1035
|
+
AND table = '${this.#adapter.escapeString(table2)}'
|
|
1036
|
+
AND is_in_primary_key = 1
|
|
1037
|
+
ORDER BY position
|
|
1038
|
+
`),
|
|
1039
|
+
this.#adapter.runQuery(`
|
|
1040
|
+
SELECT name, type_full AS type, expr AS expression
|
|
1041
|
+
FROM system.data_skipping_indices
|
|
1042
|
+
WHERE database = '${this.#adapter.escapeString(schema)}'
|
|
1043
|
+
AND table = '${this.#adapter.escapeString(table2)}'
|
|
1044
|
+
ORDER BY name
|
|
1045
|
+
`)
|
|
1046
|
+
]);
|
|
1047
|
+
const indexes2 = [];
|
|
1048
|
+
if (primaryKeyRows.length > 0) {
|
|
1049
|
+
indexes2.push({
|
|
1050
|
+
name: "PRIMARY_KEY",
|
|
1051
|
+
columns: primaryKeyRows.map((row) => row.name),
|
|
1052
|
+
type: "PRIMARY_KEY"
|
|
1053
|
+
});
|
|
1054
|
+
}
|
|
1055
|
+
indexes2.push(
|
|
1056
|
+
...indexRows.map((row) => ({
|
|
1057
|
+
name: row.name,
|
|
1058
|
+
columns: splitTopLevelExpressions(row.expression),
|
|
1059
|
+
type: row.type
|
|
1060
|
+
}))
|
|
1061
|
+
);
|
|
1062
|
+
return indexes2;
|
|
1063
|
+
}
|
|
1064
|
+
};
|
|
1065
|
+
function splitTopLevelExpressions(expression) {
|
|
1066
|
+
const parts = [];
|
|
1067
|
+
let start = 0;
|
|
1068
|
+
let depth = 0;
|
|
1069
|
+
for (let index2 = 0; index2 < expression.length; index2++) {
|
|
1070
|
+
const character = expression[index2];
|
|
1071
|
+
if (character === "(" || character === "[") depth++;
|
|
1072
|
+
if (character === ")" || character === "]") depth--;
|
|
1073
|
+
if (character === "," && depth === 0) {
|
|
1074
|
+
parts.push(expression.slice(start, index2).trim());
|
|
1075
|
+
start = index2 + 1;
|
|
1076
|
+
}
|
|
1077
|
+
}
|
|
1078
|
+
parts.push(expression.slice(start).trim());
|
|
1079
|
+
return parts.filter(Boolean);
|
|
1080
|
+
}
|
|
1081
|
+
|
|
1082
|
+
// packages/text2sql/src/lib/adapters/groundings/info.grounding.ts
|
|
1083
|
+
var InfoGrounding = class extends AbstractGrounding {
|
|
1084
|
+
constructor(config = {}) {
|
|
1085
|
+
super("dialectInfo", "info");
|
|
1086
|
+
}
|
|
1087
|
+
/**
|
|
1088
|
+
* Execute the grounding process.
|
|
1089
|
+
* Writes database info to ctx.info.
|
|
1090
|
+
*/
|
|
1091
|
+
async execute(ctx) {
|
|
1092
|
+
ctx.info = await this.collectInfo();
|
|
1093
|
+
}
|
|
1094
|
+
};
|
|
1095
|
+
|
|
1096
|
+
// packages/text2sql/src/lib/adapters/clickhouse/info.clickhouse.grounding.ts
|
|
1097
|
+
var ClickHouseInfoGrounding = class extends InfoGrounding {
|
|
1098
|
+
#adapter;
|
|
1099
|
+
constructor(adapter, config = {}) {
|
|
1100
|
+
super(config);
|
|
1101
|
+
this.#adapter = adapter;
|
|
1102
|
+
}
|
|
1103
|
+
async collectInfo() {
|
|
1104
|
+
const rows = await this.#adapter.runQuery(
|
|
1105
|
+
"SELECT version() AS version, currentDatabase() AS database"
|
|
1106
|
+
);
|
|
1107
|
+
const row = rows[0];
|
|
1108
|
+
return {
|
|
1109
|
+
dialect: "clickhouse",
|
|
1110
|
+
version: row?.version,
|
|
1111
|
+
database: row?.database || void 0,
|
|
1112
|
+
details: {
|
|
1113
|
+
identifierQuote: "`",
|
|
1114
|
+
qualifiedTable: "database.table"
|
|
1115
|
+
}
|
|
1116
|
+
};
|
|
1117
|
+
}
|
|
1118
|
+
};
|
|
1119
|
+
|
|
1120
|
+
// packages/text2sql/src/lib/adapters/groundings/row-count.grounding.ts
|
|
1121
|
+
var RowCountGrounding = class extends AbstractGrounding {
|
|
1122
|
+
constructor(config = {}) {
|
|
1123
|
+
super("rowCount", "row_counts");
|
|
1124
|
+
}
|
|
1125
|
+
/**
|
|
1126
|
+
* Execute the grounding process.
|
|
1127
|
+
* Annotates tables in ctx.tables with row counts and size hints.
|
|
1128
|
+
*/
|
|
1129
|
+
async execute(ctx) {
|
|
1130
|
+
const total = ctx.tables.length;
|
|
1131
|
+
for (let i = 0; i < ctx.tables.length; i++) {
|
|
1132
|
+
const table2 = ctx.tables[i];
|
|
1133
|
+
ctx.onProgress({
|
|
1134
|
+
type: "phase:progress",
|
|
1135
|
+
phase: "row_counts",
|
|
1136
|
+
table: table2.name,
|
|
1137
|
+
message: `Counting rows in ${table2.name}...`,
|
|
1138
|
+
current: i + 1,
|
|
1139
|
+
total
|
|
1140
|
+
});
|
|
1141
|
+
const count = await this.getRowCount(table2.name);
|
|
1142
|
+
if (count != null) {
|
|
1143
|
+
table2.rowCount = count;
|
|
1144
|
+
table2.sizeHint = this.#classifyRowCount(count);
|
|
1145
|
+
}
|
|
1146
|
+
}
|
|
1147
|
+
}
|
|
1148
|
+
/**
|
|
1149
|
+
* Classify row count into a size hint category.
|
|
1150
|
+
*/
|
|
1151
|
+
#classifyRowCount(count) {
|
|
1152
|
+
if (count < 100) return "tiny";
|
|
1153
|
+
if (count < 1e3) return "small";
|
|
1154
|
+
if (count < 1e4) return "medium";
|
|
1155
|
+
if (count < 1e5) return "large";
|
|
1156
|
+
return "huge";
|
|
1157
|
+
}
|
|
1158
|
+
};
|
|
1159
|
+
|
|
1160
|
+
// packages/text2sql/src/lib/adapters/clickhouse/row-count.clickhouse.grounding.ts
|
|
1161
|
+
var ClickHouseRowCountGrounding = class extends RowCountGrounding {
|
|
1162
|
+
#adapter;
|
|
1163
|
+
constructor(adapter, config = {}) {
|
|
1164
|
+
super(config);
|
|
1165
|
+
this.#adapter = adapter;
|
|
1166
|
+
}
|
|
1167
|
+
async getRowCount(tableName) {
|
|
1168
|
+
const { schema, table: table2 } = this.#adapter.parseTableName(tableName);
|
|
1169
|
+
const relation = schema ? `${this.#adapter.quoteIdentifier(schema)}.${this.#adapter.quoteIdentifier(table2)}` : this.#adapter.quoteIdentifier(table2);
|
|
1170
|
+
try {
|
|
1171
|
+
const rows = await this.#adapter.runQuery(`SELECT count() AS count FROM ${relation}`);
|
|
1172
|
+
return toNumber(rows[0]?.count);
|
|
1173
|
+
} catch {
|
|
1174
|
+
return void 0;
|
|
1175
|
+
}
|
|
1176
|
+
}
|
|
1177
|
+
};
|
|
1178
|
+
function toNumber(value) {
|
|
1179
|
+
if (typeof value === "number" && Number.isFinite(value)) return value;
|
|
1180
|
+
if (typeof value === "bigint") return Number(value);
|
|
1181
|
+
if (typeof value === "string" && value.trim()) {
|
|
1182
|
+
const parsed = Number(value);
|
|
1183
|
+
if (Number.isFinite(parsed)) return parsed;
|
|
1184
|
+
}
|
|
1185
|
+
return void 0;
|
|
1186
|
+
}
|
|
1187
|
+
|
|
1188
|
+
// packages/text2sql/src/lib/adapters/groundings/table.grounding.ts
|
|
1189
|
+
var TableGrounding = class extends AbstractGrounding {
|
|
1190
|
+
#filter;
|
|
1191
|
+
#columns;
|
|
1192
|
+
#forward;
|
|
1193
|
+
#backward;
|
|
1194
|
+
constructor(config = {}) {
|
|
1195
|
+
super("table", "tables");
|
|
1196
|
+
this.#filter = config.filter;
|
|
1197
|
+
this.#columns = config.columns;
|
|
1198
|
+
this.#forward = config.forward;
|
|
1199
|
+
this.#backward = config.backward;
|
|
1200
|
+
}
|
|
1201
|
+
async contributeEntities(ctx) {
|
|
1202
|
+
await this.execute(ctx);
|
|
1203
|
+
}
|
|
1204
|
+
/**
|
|
1205
|
+
* Execute the grounding process.
|
|
1206
|
+
* Writes discovered tables and relationships to the context.
|
|
1207
|
+
*/
|
|
1208
|
+
async execute(ctx) {
|
|
1209
|
+
const seedTables = await this.applyFilter();
|
|
1210
|
+
const forward = this.#forward;
|
|
1211
|
+
const backward = this.#backward;
|
|
1212
|
+
if (!forward && !backward) {
|
|
1213
|
+
const tables3 = await Promise.all(
|
|
1214
|
+
seedTables.map(async (name, index2) => {
|
|
1215
|
+
ctx.onProgress({
|
|
1216
|
+
type: "phase:progress",
|
|
1217
|
+
phase: "tables",
|
|
1218
|
+
table: name,
|
|
1219
|
+
message: `Loading table ${name}...`,
|
|
1220
|
+
current: index2 + 1,
|
|
1221
|
+
total: seedTables.length
|
|
1222
|
+
});
|
|
1223
|
+
return applyColumnFilter(await this.getTable(name), this.#columns);
|
|
1224
|
+
})
|
|
1225
|
+
);
|
|
1226
|
+
ctx.tables.push(...tables3);
|
|
1227
|
+
return;
|
|
1228
|
+
}
|
|
1229
|
+
const tables2 = {};
|
|
1230
|
+
const allRelationships = [];
|
|
1231
|
+
const seenRelationships = /* @__PURE__ */ new Set();
|
|
1232
|
+
const forwardQueue = [];
|
|
1233
|
+
const backwardQueue = [];
|
|
1234
|
+
const forwardVisited = /* @__PURE__ */ new Set();
|
|
1235
|
+
const backwardVisited = /* @__PURE__ */ new Set();
|
|
1236
|
+
let loadedCount = 0;
|
|
1237
|
+
for (const name of seedTables) {
|
|
1238
|
+
if (forward) forwardQueue.push({ name, depth: 0 });
|
|
1239
|
+
if (backward) backwardQueue.push({ name, depth: 0 });
|
|
1240
|
+
}
|
|
1241
|
+
const forwardLimit = forward === true ? Infinity : forward || 0;
|
|
1242
|
+
while (forwardQueue.length > 0) {
|
|
1243
|
+
const item = forwardQueue.shift();
|
|
1244
|
+
if (!item) break;
|
|
1245
|
+
const { name, depth } = item;
|
|
1246
|
+
if (forwardVisited.has(name)) continue;
|
|
1247
|
+
forwardVisited.add(name);
|
|
1248
|
+
if (!tables2[name]) {
|
|
1249
|
+
loadedCount++;
|
|
1250
|
+
ctx.onProgress({
|
|
1251
|
+
type: "phase:progress",
|
|
1252
|
+
phase: "tables",
|
|
1253
|
+
table: name,
|
|
1254
|
+
message: `Loading table ${name}...`,
|
|
1255
|
+
current: loadedCount
|
|
1256
|
+
});
|
|
1257
|
+
tables2[name] = applyColumnFilter(
|
|
1258
|
+
await this.getTable(name),
|
|
1259
|
+
this.#columns
|
|
1260
|
+
);
|
|
1261
|
+
}
|
|
1262
|
+
if (depth < forwardLimit) {
|
|
1263
|
+
const rels = await this.findOutgoingRelations(name);
|
|
1264
|
+
for (const rel of rels) {
|
|
1265
|
+
if (!this.isRelationshipVisible(rel)) continue;
|
|
1266
|
+
this.addRelationship(rel, allRelationships, seenRelationships);
|
|
1267
|
+
if (!forwardVisited.has(rel.referenced_table)) {
|
|
1268
|
+
forwardQueue.push({ name: rel.referenced_table, depth: depth + 1 });
|
|
1269
|
+
}
|
|
1270
|
+
}
|
|
1271
|
+
}
|
|
1272
|
+
}
|
|
1273
|
+
const backwardLimit = backward === true ? Infinity : backward || 0;
|
|
1274
|
+
while (backwardQueue.length > 0) {
|
|
1275
|
+
const item = backwardQueue.shift();
|
|
1276
|
+
if (!item) break;
|
|
1277
|
+
const { name, depth } = item;
|
|
1278
|
+
if (backwardVisited.has(name)) continue;
|
|
1279
|
+
backwardVisited.add(name);
|
|
1280
|
+
if (!tables2[name]) {
|
|
1281
|
+
loadedCount++;
|
|
1282
|
+
ctx.onProgress({
|
|
1283
|
+
type: "phase:progress",
|
|
1284
|
+
phase: "tables",
|
|
1285
|
+
table: name,
|
|
1286
|
+
message: `Loading table ${name}...`,
|
|
1287
|
+
current: loadedCount
|
|
1288
|
+
});
|
|
1289
|
+
tables2[name] = applyColumnFilter(
|
|
1290
|
+
await this.getTable(name),
|
|
1291
|
+
this.#columns
|
|
1292
|
+
);
|
|
1293
|
+
}
|
|
1294
|
+
if (depth < backwardLimit) {
|
|
1295
|
+
const rels = await this.findIncomingRelations(name);
|
|
1296
|
+
for (const rel of rels) {
|
|
1297
|
+
if (!this.isRelationshipVisible(rel)) continue;
|
|
1298
|
+
this.addRelationship(rel, allRelationships, seenRelationships);
|
|
1299
|
+
if (!backwardVisited.has(rel.table)) {
|
|
1300
|
+
backwardQueue.push({ name: rel.table, depth: depth + 1 });
|
|
1301
|
+
}
|
|
1302
|
+
}
|
|
1303
|
+
}
|
|
1304
|
+
}
|
|
1305
|
+
const tablesList = Object.values(tables2);
|
|
1306
|
+
ctx.tables.push(...tablesList);
|
|
1307
|
+
ctx.relationships.push(...allRelationships);
|
|
1308
|
+
}
|
|
1309
|
+
/**
|
|
1310
|
+
* Apply the filter to get seed table names.
|
|
1311
|
+
* If filter is an explicit array, skip querying all table names.
|
|
1312
|
+
*/
|
|
1313
|
+
async applyFilter() {
|
|
1314
|
+
const filter = this.#filter;
|
|
1315
|
+
if (Array.isArray(filter)) {
|
|
1316
|
+
return filter;
|
|
1317
|
+
}
|
|
1318
|
+
const names = await this.getAllTableNames();
|
|
1319
|
+
if (!filter) {
|
|
1320
|
+
return names;
|
|
1321
|
+
}
|
|
1322
|
+
if (filter instanceof RegExp) {
|
|
1323
|
+
return names.filter((name) => filter.test(name));
|
|
1324
|
+
}
|
|
1325
|
+
return names.filter(filter);
|
|
1326
|
+
}
|
|
1327
|
+
/**
|
|
1328
|
+
* Add a relationship to the collection, deduplicating by key.
|
|
1329
|
+
*/
|
|
1330
|
+
addRelationship(rel, all, seen) {
|
|
1331
|
+
const key = `${rel.table}:${rel.from.join(",")}:${rel.referenced_table}:${rel.to.join(",")}`;
|
|
1332
|
+
if (!seen.has(key)) {
|
|
1333
|
+
seen.add(key);
|
|
1334
|
+
all.push(rel);
|
|
1335
|
+
}
|
|
1336
|
+
}
|
|
1337
|
+
isRelationshipVisible(rel) {
|
|
1338
|
+
return this.areColumnsVisible(rel.from, this.#columns?.[rel.table]) && this.areColumnsVisible(rel.to, this.#columns?.[rel.referenced_table]);
|
|
1339
|
+
}
|
|
1340
|
+
areColumnsVisible(names, filter) {
|
|
1341
|
+
if (!filter) return true;
|
|
1342
|
+
return filterColumns(
|
|
1343
|
+
names.map((name) => ({ name })),
|
|
1344
|
+
filter
|
|
1345
|
+
).length === names.length;
|
|
1346
|
+
}
|
|
1347
|
+
};
|
|
1348
|
+
|
|
1349
|
+
// packages/text2sql/src/lib/adapters/clickhouse/table.clickhouse.grounding.ts
|
|
1350
|
+
var ClickHouseTableGrounding = class extends TableGrounding {
|
|
1351
|
+
#adapter;
|
|
1352
|
+
constructor(adapter, config = {}) {
|
|
1353
|
+
super(config);
|
|
1354
|
+
this.#adapter = adapter;
|
|
1355
|
+
}
|
|
1356
|
+
async getAllTableNames() {
|
|
1357
|
+
const rows = await this.#adapter.runQuery(`
|
|
1358
|
+
SELECT concat(database, '.', name) AS name
|
|
1359
|
+
FROM system.tables
|
|
1360
|
+
WHERE database NOT IN ('system', 'information_schema', 'INFORMATION_SCHEMA')
|
|
1361
|
+
AND is_temporary = 0
|
|
1362
|
+
AND engine != 'View'
|
|
1363
|
+
ORDER BY database, name
|
|
1364
|
+
`);
|
|
1365
|
+
return rows.map((row) => row.name);
|
|
1366
|
+
}
|
|
1367
|
+
async getTable(tableName) {
|
|
1368
|
+
const { schema, table: table2 } = this.#adapter.parseTableName(tableName);
|
|
1369
|
+
if (!schema) {
|
|
1370
|
+
throw new Error(
|
|
1371
|
+
`ClickHouse table grounding requires a qualified table name: ${tableName}`
|
|
1372
|
+
);
|
|
1373
|
+
}
|
|
1374
|
+
const rows = await this.#adapter.runQuery(`
|
|
1375
|
+
SELECT name, type
|
|
1376
|
+
FROM system.columns
|
|
1377
|
+
WHERE database = '${this.#adapter.escapeString(schema)}'
|
|
1378
|
+
AND table = '${this.#adapter.escapeString(table2)}'
|
|
1379
|
+
ORDER BY position
|
|
1380
|
+
`);
|
|
1381
|
+
return {
|
|
1382
|
+
name: tableName,
|
|
1383
|
+
schema,
|
|
1384
|
+
rawName: table2,
|
|
1385
|
+
columns: rows.map((row) => ({ name: row.name, type: row.type }))
|
|
1386
|
+
};
|
|
1387
|
+
}
|
|
1388
|
+
async findOutgoingRelations() {
|
|
1389
|
+
return [];
|
|
1390
|
+
}
|
|
1391
|
+
async findIncomingRelations() {
|
|
1392
|
+
return [];
|
|
1393
|
+
}
|
|
1394
|
+
};
|
|
1395
|
+
|
|
1396
|
+
// packages/text2sql/src/lib/adapters/groundings/view.grounding.ts
|
|
1397
|
+
var ViewGrounding = class extends AbstractGrounding {
|
|
1398
|
+
#filter;
|
|
1399
|
+
#columns;
|
|
1400
|
+
includeDefinition;
|
|
1401
|
+
constructor(config = {}) {
|
|
1402
|
+
super("view", "views");
|
|
1403
|
+
this.#filter = config.filter;
|
|
1404
|
+
this.#columns = config.columns;
|
|
1405
|
+
this.includeDefinition = config.includeDefinition ?? true;
|
|
1406
|
+
}
|
|
1407
|
+
async contributeEntities(ctx) {
|
|
1408
|
+
await this.execute(ctx);
|
|
1409
|
+
}
|
|
1410
|
+
/**
|
|
1411
|
+
* Execute the grounding process.
|
|
1412
|
+
* Writes discovered views to the context.
|
|
1413
|
+
*/
|
|
1414
|
+
async execute(ctx) {
|
|
1415
|
+
const viewNames = await this.applyFilter();
|
|
1416
|
+
const views2 = await Promise.all(
|
|
1417
|
+
viewNames.map(async (name, index2) => {
|
|
1418
|
+
ctx.onProgress({
|
|
1419
|
+
type: "phase:progress",
|
|
1420
|
+
phase: "views",
|
|
1421
|
+
table: name,
|
|
1422
|
+
message: `Loading view ${name}...`,
|
|
1423
|
+
current: index2 + 1,
|
|
1424
|
+
total: viewNames.length
|
|
1425
|
+
});
|
|
1426
|
+
return applyColumnFilter(await this.getView(name), this.#columns);
|
|
1427
|
+
})
|
|
1428
|
+
);
|
|
1429
|
+
ctx.views.push(...views2);
|
|
1430
|
+
}
|
|
1431
|
+
/**
|
|
1432
|
+
* Apply the filter to get view names.
|
|
1433
|
+
* If filter is an explicit array, skip querying all view names.
|
|
1434
|
+
*/
|
|
1435
|
+
async applyFilter() {
|
|
1436
|
+
const filter = this.#filter;
|
|
1437
|
+
if (Array.isArray(filter)) {
|
|
1438
|
+
return filter;
|
|
1439
|
+
}
|
|
1440
|
+
const names = await this.getAllViewNames();
|
|
1441
|
+
if (!filter) {
|
|
1442
|
+
return names;
|
|
1443
|
+
}
|
|
1444
|
+
if (filter instanceof RegExp) {
|
|
1445
|
+
return names.filter((name) => filter.test(name));
|
|
1446
|
+
}
|
|
1447
|
+
return names.filter(filter);
|
|
1448
|
+
}
|
|
1449
|
+
};
|
|
1450
|
+
|
|
1451
|
+
// packages/text2sql/src/lib/adapters/clickhouse/view.clickhouse.grounding.ts
|
|
1452
|
+
var ClickHouseViewGrounding = class extends ViewGrounding {
|
|
1453
|
+
#adapter;
|
|
1454
|
+
constructor(adapter, config = {}) {
|
|
1455
|
+
super(config);
|
|
1456
|
+
this.#adapter = adapter;
|
|
1457
|
+
}
|
|
1458
|
+
async getAllViewNames() {
|
|
1459
|
+
const rows = await this.#adapter.runQuery(`
|
|
1460
|
+
SELECT concat(database, '.', name) AS name
|
|
1461
|
+
FROM system.tables
|
|
1462
|
+
WHERE database NOT IN ('system', 'information_schema', 'INFORMATION_SCHEMA')
|
|
1463
|
+
AND is_temporary = 0
|
|
1464
|
+
AND endsWith(engine, 'View')
|
|
1465
|
+
ORDER BY database, name
|
|
1466
|
+
`);
|
|
1467
|
+
return rows.map((row) => row.name);
|
|
1468
|
+
}
|
|
1469
|
+
async getView(viewName) {
|
|
1470
|
+
const { schema, table: table2 } = this.#adapter.parseTableName(viewName);
|
|
1471
|
+
if (!schema) {
|
|
1472
|
+
throw new Error(
|
|
1473
|
+
`ClickHouse view grounding requires a qualified view name: ${viewName}`
|
|
1474
|
+
);
|
|
1475
|
+
}
|
|
1476
|
+
const [columns, definitionRows] = await Promise.all([
|
|
1477
|
+
this.#adapter.runQuery(`
|
|
1478
|
+
SELECT name, type
|
|
1479
|
+
FROM system.columns
|
|
1480
|
+
WHERE database = '${this.#adapter.escapeString(schema)}'
|
|
1481
|
+
AND table = '${this.#adapter.escapeString(table2)}'
|
|
1482
|
+
ORDER BY position
|
|
1483
|
+
`),
|
|
1484
|
+
this.includeDefinition ? this.#adapter.runQuery(`
|
|
1485
|
+
SELECT create_table_query AS definition
|
|
1486
|
+
FROM system.tables
|
|
1487
|
+
WHERE database = '${this.#adapter.escapeString(schema)}'
|
|
1488
|
+
AND name = '${this.#adapter.escapeString(table2)}'
|
|
1489
|
+
AND endsWith(engine, 'View')
|
|
1490
|
+
`) : Promise.resolve([])
|
|
1491
|
+
]);
|
|
1492
|
+
return {
|
|
1493
|
+
name: viewName,
|
|
1494
|
+
schema,
|
|
1495
|
+
rawName: table2,
|
|
1496
|
+
definition: definitionRows[0]?.definition,
|
|
1497
|
+
columns: columns.map((column2) => ({
|
|
1498
|
+
name: column2.name,
|
|
1499
|
+
type: column2.type
|
|
1500
|
+
}))
|
|
1501
|
+
};
|
|
1502
|
+
}
|
|
1503
|
+
};
|
|
1504
|
+
|
|
1505
|
+
// packages/text2sql/src/lib/adapters/clickhouse/index.ts
|
|
1506
|
+
function tables(config = {}) {
|
|
1507
|
+
return (adapter) => new ClickHouseTableGrounding(adapter, config);
|
|
1508
|
+
}
|
|
1509
|
+
function views(config = {}) {
|
|
1510
|
+
return (adapter) => new ClickHouseViewGrounding(adapter, config);
|
|
1511
|
+
}
|
|
1512
|
+
function info(config = {}) {
|
|
1513
|
+
return (adapter) => new ClickHouseInfoGrounding(adapter, config);
|
|
1514
|
+
}
|
|
1515
|
+
function indexes(config = {}) {
|
|
1516
|
+
return (adapter) => new ClickHouseIndexesGrounding(adapter, config);
|
|
1517
|
+
}
|
|
1518
|
+
function rowCount(config = {}) {
|
|
1519
|
+
return (adapter) => new ClickHouseRowCountGrounding(adapter, config);
|
|
1520
|
+
}
|
|
1521
|
+
function constraints(config = {}) {
|
|
1522
|
+
return (adapter) => new ClickHouseConstraintGrounding(adapter, config);
|
|
1523
|
+
}
|
|
1524
|
+
var clickhouse_default = {
|
|
1525
|
+
ClickHouse,
|
|
1526
|
+
constraints,
|
|
1527
|
+
indexes,
|
|
1528
|
+
info,
|
|
1529
|
+
rowCount,
|
|
1530
|
+
tables,
|
|
1531
|
+
views
|
|
1532
|
+
};
|
|
1533
|
+
export {
|
|
1534
|
+
ClickHouse,
|
|
1535
|
+
ClickHouseSqlPolicyAnalyzer,
|
|
1536
|
+
constraints,
|
|
1537
|
+
clickhouse_default as default,
|
|
1538
|
+
indexes,
|
|
1539
|
+
info,
|
|
1540
|
+
rowCount,
|
|
1541
|
+
tables,
|
|
1542
|
+
views
|
|
1543
|
+
};
|
|
1544
|
+
//# sourceMappingURL=index.js.map
|