@deepagents/text2sql 7.0.2 → 7.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +63 -75
- package/dist/bin/sql.js +82 -599
- package/dist/eval/src/evals/models.d.ts +12 -0
- package/dist/eval/src/evals/models.d.ts.map +1 -0
- package/dist/eval/src/evals/sql-create-context/sql-create-context.eval.d.ts +2 -0
- package/dist/eval/src/evals/sql-create-context/sql-create-context.eval.d.ts.map +1 -0
- package/dist/index.d.ts +0 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +920 -706
- package/dist/index.js.map +4 -4
- package/dist/lib/adapter-index.d.ts +0 -2
- package/dist/lib/adapter-index.d.ts.map +1 -1
- package/dist/lib/adapters/adapter.d.ts +23 -1
- package/dist/lib/adapters/adapter.d.ts.map +1 -1
- package/dist/lib/adapters/bigquery/bigquery-fk.d.ts +0 -11
- package/dist/lib/adapters/bigquery/bigquery-fk.d.ts.map +1 -1
- package/dist/lib/adapters/bigquery/bigquery-rows.d.ts +22 -0
- package/dist/lib/adapters/bigquery/bigquery-rows.d.ts.map +1 -0
- package/dist/lib/adapters/bigquery/bigquery.d.ts +1 -1
- package/dist/lib/adapters/bigquery/bigquery.d.ts.map +1 -1
- package/dist/lib/adapters/bigquery/constraint.bigquery.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/bigquery/index.d.ts +6 -13
- package/dist/lib/adapters/bigquery/index.d.ts.map +1 -1
- package/dist/lib/adapters/bigquery/index.js +244 -51
- package/dist/lib/adapters/bigquery/index.js.map +4 -4
- package/dist/lib/adapters/bigquery/indexes.bigquery.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/bigquery/row-count.bigquery.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/bigquery/table.bigquery.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/bigquery/view.bigquery.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/clickhouse/clickhouse-rows.d.ts +7 -0
- package/dist/lib/adapters/clickhouse/clickhouse-rows.d.ts.map +1 -0
- package/dist/lib/adapters/clickhouse/clickhouse.d.ts +1 -1
- package/dist/lib/adapters/clickhouse/clickhouse.d.ts.map +1 -1
- package/dist/lib/adapters/clickhouse/constraint.clickhouse.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/clickhouse/index.js +133 -44
- package/dist/lib/adapters/clickhouse/index.js.map +4 -4
- package/dist/lib/adapters/clickhouse/indexes.clickhouse.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/clickhouse/info.clickhouse.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/clickhouse/row-count.clickhouse.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/clickhouse/table.clickhouse.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/clickhouse/view.clickhouse.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/duckdb/duckdb.d.ts +1 -1
- package/dist/lib/adapters/duckdb/duckdb.d.ts.map +1 -1
- package/dist/lib/adapters/duckdb/duckdb.groundings.d.ts.map +1 -1
- package/dist/lib/adapters/duckdb/index.d.ts +8 -10
- package/dist/lib/adapters/duckdb/index.d.ts.map +1 -1
- package/dist/lib/adapters/duckdb/index.js +1549 -1412
- package/dist/lib/adapters/duckdb/index.js.map +4 -4
- package/dist/lib/adapters/groundings/index.js.map +1 -1
- package/dist/lib/adapters/groundings/rows.d.ts +32 -0
- package/dist/lib/adapters/groundings/rows.d.ts.map +1 -0
- package/dist/lib/adapters/mysql/column-stats.mysql.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/mysql/column-values.mysql.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/mysql/constraint.mysql.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/mysql/index.d.ts +2 -4
- package/dist/lib/adapters/mysql/index.d.ts.map +1 -1
- package/dist/lib/adapters/mysql/index.js +230 -76
- package/dist/lib/adapters/mysql/index.js.map +4 -4
- package/dist/lib/adapters/mysql/indexes.mysql.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/mysql/info.mysql.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/mysql/mysql-rows.d.ts +11 -0
- package/dist/lib/adapters/mysql/mysql-rows.d.ts.map +1 -0
- package/dist/lib/adapters/mysql/mysql.d.ts +1 -1
- package/dist/lib/adapters/mysql/mysql.d.ts.map +1 -1
- package/dist/lib/adapters/mysql/row-count.mysql.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/mysql/table.mysql.grounding.d.ts +3 -2
- package/dist/lib/adapters/mysql/table.mysql.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/mysql/view.mysql.grounding.d.ts +2 -2
- package/dist/lib/adapters/mysql/view.mysql.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/postgres/column-stats.postgres.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/postgres/column-values.postgres.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/postgres/constraint.postgres.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/postgres/index.d.ts +2 -4
- package/dist/lib/adapters/postgres/index.d.ts.map +1 -1
- package/dist/lib/adapters/postgres/index.js +266 -487
- package/dist/lib/adapters/postgres/index.js.map +4 -4
- package/dist/lib/adapters/postgres/indexes.postgres.grounding.d.ts +1 -5
- package/dist/lib/adapters/postgres/indexes.postgres.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/postgres/info.postgres.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/postgres/postgres-rows.d.ts +7 -0
- package/dist/lib/adapters/postgres/postgres-rows.d.ts.map +1 -0
- package/dist/lib/adapters/postgres/postgres.d.ts +6 -1
- package/dist/lib/adapters/postgres/postgres.d.ts.map +1 -1
- package/dist/lib/adapters/postgres/row-count.postgres.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/postgres/table.postgres.grounding.d.ts +5 -4
- package/dist/lib/adapters/postgres/table.postgres.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/postgres/view.postgres.grounding.d.ts +3 -3
- package/dist/lib/adapters/postgres/view.postgres.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/posthog/definitions.posthog.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/posthog/index.d.ts +4 -5
- package/dist/lib/adapters/posthog/index.d.ts.map +1 -1
- package/dist/lib/adapters/posthog/index.js +349 -287
- package/dist/lib/adapters/posthog/index.js.map +4 -4
- package/dist/lib/adapters/posthog/posthog.d.ts +2 -2
- package/dist/lib/adapters/posthog/posthog.d.ts.map +1 -1
- package/dist/lib/adapters/posthog/posthog.sql-policy.d.ts.map +1 -1
- package/dist/lib/adapters/posthog/schema.posthog.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/posthog/transport.d.ts.map +1 -1
- package/dist/lib/adapters/posthog/types.d.ts +16 -6
- package/dist/lib/adapters/posthog/types.d.ts.map +1 -1
- package/dist/lib/adapters/spreadsheet/index.js +117 -38
- package/dist/lib/adapters/spreadsheet/index.js.map +4 -4
- package/dist/lib/adapters/sqlite/column-stats.sqlite.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/sqlite/column-values.sqlite.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/sqlite/constraint.sqlite.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/sqlite/index.js +136 -52
- package/dist/lib/adapters/sqlite/index.js.map +4 -4
- package/dist/lib/adapters/sqlite/indexes.sqlite.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/sqlite/info.sqlite.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/sqlite/row-count.sqlite.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/sqlite/sqlite-rows.d.ts +41 -0
- package/dist/lib/adapters/sqlite/sqlite-rows.d.ts.map +1 -0
- package/dist/lib/adapters/sqlite/sqlite.d.ts +1 -1
- package/dist/lib/adapters/sqlite/sqlite.d.ts.map +1 -1
- package/dist/lib/adapters/sqlite/table.sqlite.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/sqlite/view.sqlite.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/sqlserver/column-stats.sqlserver.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/sqlserver/column-values.sqlserver.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/sqlserver/constraint.sqlserver.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/sqlserver/index.d.ts +2 -4
- package/dist/lib/adapters/sqlserver/index.d.ts.map +1 -1
- package/dist/lib/adapters/sqlserver/index.js +255 -503
- package/dist/lib/adapters/sqlserver/index.js.map +4 -4
- package/dist/lib/adapters/sqlserver/indexes.sqlserver.grounding.d.ts +1 -5
- package/dist/lib/adapters/sqlserver/indexes.sqlserver.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/sqlserver/info.sqlserver.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/sqlserver/row-count.sqlserver.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/sqlserver/sqlserver-rows.d.ts +7 -0
- package/dist/lib/adapters/sqlserver/sqlserver-rows.d.ts.map +1 -0
- package/dist/lib/adapters/sqlserver/sqlserver.d.ts +6 -1
- package/dist/lib/adapters/sqlserver/sqlserver.d.ts.map +1 -1
- package/dist/lib/adapters/sqlserver/table.sqlserver.grounding.d.ts +5 -4
- package/dist/lib/adapters/sqlserver/table.sqlserver.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/sqlserver/view.sqlserver.grounding.d.ts +3 -3
- package/dist/lib/adapters/sqlserver/view.sqlserver.grounding.d.ts.map +1 -1
- package/dist/lib/agents/question.agent.d.ts.map +1 -1
- package/dist/lib/agents/sql-transform-plugins.d.ts.map +1 -1
- package/dist/lib/agents/suggestions.agents.d.ts.map +1 -1
- package/dist/lib/checkpoint.d.ts +34 -21
- package/dist/lib/checkpoint.d.ts.map +1 -1
- package/dist/lib/file-cache.d.ts +6 -1
- package/dist/lib/file-cache.d.ts.map +1 -1
- package/dist/lib/fs/columns.d.ts +14 -0
- package/dist/lib/fs/columns.d.ts.map +1 -0
- package/dist/lib/fs/mssql/mssql-fs.d.ts +4 -4
- package/dist/lib/fs/mssql/mssql-fs.d.ts.map +1 -1
- package/dist/lib/fs/postgres/postgres-fs.d.ts +4 -4
- package/dist/lib/fs/postgres/postgres-fs.d.ts.map +1 -1
- package/dist/lib/fs/sqlite/sqlite-fs.d.ts +4 -4
- package/dist/lib/fs/sqlite/sqlite-fs.d.ts.map +1 -1
- package/dist/lib/index-cache.d.ts +6 -6
- package/dist/lib/index-cache.d.ts.map +1 -1
- package/dist/lib/sql-command.d.ts +1 -4
- package/dist/lib/sql-command.d.ts.map +1 -1
- package/dist/lib/sql.d.ts +0 -5
- package/dist/lib/sql.d.ts.map +1 -1
- package/dist/lib/synthesis/decorators/deduplicated-producer.d.ts +1 -2
- package/dist/lib/synthesis/decorators/deduplicated-producer.d.ts.map +1 -1
- package/dist/lib/synthesis/decorators/filtered-producer.d.ts +0 -2
- package/dist/lib/synthesis/decorators/filtered-producer.d.ts.map +1 -1
- package/dist/lib/synthesis/decorators/validated-producer.d.ts +1 -3
- package/dist/lib/synthesis/decorators/validated-producer.d.ts.map +1 -1
- package/dist/lib/synthesis/extractors/base-contextual-extractor.d.ts.map +1 -1
- package/dist/lib/synthesis/extractors/message-extractor.d.ts.map +1 -1
- package/dist/lib/synthesis/extractors/segmented-context-extractor.d.ts.map +1 -1
- package/dist/lib/synthesis/extractors/sql-extractor.d.ts.map +1 -1
- package/dist/lib/synthesis/extractors/tool-input.d.ts +6 -0
- package/dist/lib/synthesis/extractors/tool-input.d.ts.map +1 -0
- package/dist/lib/synthesis/index.js +124 -114
- package/dist/lib/synthesis/index.js.map +4 -4
- package/dist/lib/synthesis/synthesizers/breadth-evolver.d.ts +0 -2
- package/dist/lib/synthesis/synthesizers/breadth-evolver.d.ts.map +1 -1
- package/dist/lib/synthesis/synthesizers/depth-evolver.d.ts +0 -3
- package/dist/lib/synthesis/synthesizers/depth-evolver.d.ts.map +1 -1
- package/dist/lib/synthesis/synthesizers/persona-generator.d.ts.map +1 -1
- package/dist/lib/synthesis/synthesizers/schema-synthesizer.d.ts +0 -2
- package/dist/lib/synthesis/synthesizers/schema-synthesizer.d.ts.map +1 -1
- package/dist/scripts/scripts/bigquery-test.d.ts +2 -0
- package/dist/scripts/scripts/bigquery-test.d.ts.map +1 -0
- package/package.json +4 -6
- package/dist/lib/index-lock.d.ts +0 -60
- package/dist/lib/index-lock.d.ts.map +0 -1
- package/dist/lib/prompt.d.ts +0 -2
- package/dist/lib/prompt.d.ts.map +0 -1
package/dist/bin/sql.js
CHANGED
|
@@ -3,6 +3,12 @@
|
|
|
3
3
|
// packages/text2sql/src/bin/sql.ts
|
|
4
4
|
import { cac } from "cac";
|
|
5
5
|
|
|
6
|
+
// packages/text2sql/src/lib/index-cache.ts
|
|
7
|
+
import {
|
|
8
|
+
isFragment,
|
|
9
|
+
isFragmentData
|
|
10
|
+
} from "@deepagents/context";
|
|
11
|
+
|
|
6
12
|
// packages/text2sql/src/lib/file-cache.ts
|
|
7
13
|
import { createHash, randomUUID } from "node:crypto";
|
|
8
14
|
import { existsSync } from "node:fs";
|
|
@@ -29,8 +35,14 @@ var FileCache = class {
|
|
|
29
35
|
}
|
|
30
36
|
};
|
|
31
37
|
var JsonCache = class extends FileCache {
|
|
32
|
-
|
|
38
|
+
#isValid;
|
|
39
|
+
/**
|
|
40
|
+
* @param isValid - Checks that a parsed cache file holds a T; a file that
|
|
41
|
+
* fails it is read as a miss.
|
|
42
|
+
*/
|
|
43
|
+
constructor(watermark, isValid, baseDir) {
|
|
33
44
|
super(watermark, ".json", baseDir);
|
|
45
|
+
this.#isValid = isValid;
|
|
34
46
|
}
|
|
35
47
|
async read() {
|
|
36
48
|
const content = await this.get();
|
|
@@ -38,7 +50,8 @@ var JsonCache = class extends FileCache {
|
|
|
38
50
|
return null;
|
|
39
51
|
}
|
|
40
52
|
try {
|
|
41
|
-
|
|
53
|
+
const value = JSON.parse(content);
|
|
54
|
+
return this.#isValid(value) ? value : null;
|
|
42
55
|
} catch {
|
|
43
56
|
return null;
|
|
44
57
|
}
|
|
@@ -64,57 +77,12 @@ var FileIndexCache = class {
|
|
|
64
77
|
}
|
|
65
78
|
#cacheFor(key) {
|
|
66
79
|
const watermark = this.#namespace ? `${this.#namespace}-${key}` : key;
|
|
67
|
-
return new JsonCache(watermark, this.#dir);
|
|
80
|
+
return new JsonCache(watermark, isFragmentList, this.#dir);
|
|
68
81
|
}
|
|
69
82
|
};
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
import { mkdir as mkdir2 } from "node:fs/promises";
|
|
74
|
-
import { tmpdir as tmpdir2 } from "node:os";
|
|
75
|
-
import * as path2 from "node:path";
|
|
76
|
-
import { lock as acquireLock } from "proper-lockfile";
|
|
77
|
-
var DEFAULT_STALE_MS = 3e4;
|
|
78
|
-
var DEFAULT_RETRIES = {
|
|
79
|
-
retries: 10,
|
|
80
|
-
factor: 2,
|
|
81
|
-
minTimeout: 100,
|
|
82
|
-
maxTimeout: 2e3
|
|
83
|
-
};
|
|
84
|
-
var FileIndexLock = class {
|
|
85
|
-
#dir;
|
|
86
|
-
#namespace;
|
|
87
|
-
#stale;
|
|
88
|
-
#retries;
|
|
89
|
-
constructor(options = {}) {
|
|
90
|
-
this.#dir = options.dir ?? tmpdir2();
|
|
91
|
-
this.#namespace = options.namespace;
|
|
92
|
-
this.#stale = options.stale ?? DEFAULT_STALE_MS;
|
|
93
|
-
this.#retries = options.retries ?? DEFAULT_RETRIES;
|
|
94
|
-
}
|
|
95
|
-
async run(key, fn) {
|
|
96
|
-
await mkdir2(this.#dir, { recursive: true });
|
|
97
|
-
const release = await acquireLock(this.#lockPathFor(key), {
|
|
98
|
-
realpath: false,
|
|
99
|
-
stale: this.#stale,
|
|
100
|
-
retries: this.#retries
|
|
101
|
-
});
|
|
102
|
-
try {
|
|
103
|
-
return await fn();
|
|
104
|
-
} finally {
|
|
105
|
-
await release().catch(() => {
|
|
106
|
-
});
|
|
107
|
-
}
|
|
108
|
-
}
|
|
109
|
-
#lockPathFor(key) {
|
|
110
|
-
const watermark = this.#namespace ? `${this.#namespace}-${key}` : key;
|
|
111
|
-
const hash = createHash2("md5").update(watermark).digest("hex");
|
|
112
|
-
return path2.join(this.#dir, `text2sql-lock-${hash}`);
|
|
113
|
-
}
|
|
114
|
-
};
|
|
115
|
-
|
|
116
|
-
// packages/text2sql/src/lib/sql.ts
|
|
117
|
-
import "@deepagents/agent";
|
|
83
|
+
function isFragmentList(value) {
|
|
84
|
+
return Array.isArray(value) && value.every((item) => isFragment(item) && isFragmentData(item));
|
|
85
|
+
}
|
|
118
86
|
|
|
119
87
|
// packages/text2sql/src/lib/adapter-index.ts
|
|
120
88
|
import { fragment } from "@deepagents/context";
|
|
@@ -141,6 +109,7 @@ function validateAdapterNames(names) {
|
|
|
141
109
|
|
|
142
110
|
// packages/text2sql/src/lib/adapters/adapter.ts
|
|
143
111
|
import { format as formatSql } from "sql-formatter";
|
|
112
|
+
import { z } from "zod";
|
|
144
113
|
|
|
145
114
|
// packages/text2sql/src/lib/agents/exceptions.ts
|
|
146
115
|
var sqlValidationMarker = /* @__PURE__ */ Symbol("SQLValidationError");
|
|
@@ -238,7 +207,6 @@ function databaseNameFragment(name) {
|
|
|
238
207
|
var AdapterIndexer = class {
|
|
239
208
|
#adapters;
|
|
240
209
|
#cache;
|
|
241
|
-
#lock;
|
|
242
210
|
constructor(options) {
|
|
243
211
|
const adapterNames = Object.keys(options.adapters);
|
|
244
212
|
if (adapterNames.length === 0) {
|
|
@@ -247,7 +215,6 @@ var AdapterIndexer = class {
|
|
|
247
215
|
validateAdapterNames(adapterNames);
|
|
248
216
|
this.#adapters = options.adapters;
|
|
249
217
|
this.#cache = options.cache;
|
|
250
|
-
this.#lock = options.lock;
|
|
251
218
|
}
|
|
252
219
|
async index(options = {}) {
|
|
253
220
|
const progress = timestampProgressHandler(options.onProgress);
|
|
@@ -305,41 +272,15 @@ var AdapterIndexer = class {
|
|
|
305
272
|
message: `Indexing adapter "${name}"...`
|
|
306
273
|
});
|
|
307
274
|
const cache = this.#cache;
|
|
308
|
-
const key = name;
|
|
309
|
-
const emitCacheHit = () => progress({
|
|
310
|
-
type: "adapter:cache-hit",
|
|
311
|
-
adapter: name,
|
|
312
|
-
message: `Using cached index for adapter "${name}".`,
|
|
313
|
-
cached: true
|
|
314
|
-
});
|
|
315
|
-
const emitCacheMiss = () => progress({
|
|
316
|
-
type: "adapter:cache-miss",
|
|
317
|
-
adapter: name,
|
|
318
|
-
message: `No cached index for adapter "${name}".`,
|
|
319
|
-
cached: false
|
|
320
|
-
});
|
|
321
|
-
const introspect = async () => {
|
|
322
|
-
const ctx = createGroundingContext({
|
|
323
|
-
onProgress: (event) => progress(adapterProgressEvent(name, event))
|
|
324
|
-
});
|
|
325
|
-
const fragments = await adapter.introspect(ctx);
|
|
326
|
-
await cache?.write(key, fragments);
|
|
327
|
-
return fragments;
|
|
328
|
-
};
|
|
329
|
-
let servedFromCache = false;
|
|
330
|
-
const populate = async () => {
|
|
331
|
-
const recheck = await cache?.read(key);
|
|
332
|
-
if (recheck) {
|
|
333
|
-
servedFromCache = true;
|
|
334
|
-
return recheck;
|
|
335
|
-
}
|
|
336
|
-
if (cache) emitCacheMiss();
|
|
337
|
-
return introspect();
|
|
338
|
-
};
|
|
339
275
|
try {
|
|
340
|
-
const cached = await cache?.read(
|
|
276
|
+
const cached = await cache?.read(name);
|
|
341
277
|
if (cached) {
|
|
342
|
-
|
|
278
|
+
progress({
|
|
279
|
+
type: "adapter:cache-hit",
|
|
280
|
+
adapter: name,
|
|
281
|
+
message: `Using cached index for adapter "${name}".`,
|
|
282
|
+
cached: true
|
|
283
|
+
});
|
|
343
284
|
progress({
|
|
344
285
|
type: "adapter:end",
|
|
345
286
|
adapter: name,
|
|
@@ -348,13 +289,24 @@ var AdapterIndexer = class {
|
|
|
348
289
|
});
|
|
349
290
|
return cached;
|
|
350
291
|
}
|
|
351
|
-
|
|
352
|
-
|
|
292
|
+
if (cache) {
|
|
293
|
+
progress({
|
|
294
|
+
type: "adapter:cache-miss",
|
|
295
|
+
adapter: name,
|
|
296
|
+
message: `No cached index for adapter "${name}".`,
|
|
297
|
+
cached: false
|
|
298
|
+
});
|
|
299
|
+
}
|
|
300
|
+
const ctx = createGroundingContext({
|
|
301
|
+
onProgress: (event) => progress(adapterProgressEvent(name, event))
|
|
302
|
+
});
|
|
303
|
+
const fragments = await adapter.introspect(ctx);
|
|
304
|
+
await cache?.write(name, fragments);
|
|
353
305
|
progress({
|
|
354
306
|
type: "adapter:end",
|
|
355
307
|
adapter: name,
|
|
356
308
|
message: `Finished indexing adapter "${name}".`,
|
|
357
|
-
cached:
|
|
309
|
+
cached: false
|
|
358
310
|
});
|
|
359
311
|
return fragments;
|
|
360
312
|
} catch (error) {
|
|
@@ -399,456 +351,6 @@ function errorMessage(error) {
|
|
|
399
351
|
return error instanceof Error ? error.message : String(error);
|
|
400
352
|
}
|
|
401
353
|
|
|
402
|
-
// packages/text2sql/src/lib/agents/sql.agent.ts
|
|
403
|
-
import { groq } from "@ai-sdk/groq";
|
|
404
|
-
import {
|
|
405
|
-
APICallError,
|
|
406
|
-
JSONParseError,
|
|
407
|
-
NoContentGeneratedError,
|
|
408
|
-
NoObjectGeneratedError,
|
|
409
|
-
NoOutputGeneratedError,
|
|
410
|
-
TypeValidationError,
|
|
411
|
-
defaultSettingsMiddleware,
|
|
412
|
-
wrapLanguageModel
|
|
413
|
-
} from "ai";
|
|
414
|
-
import dedent from "dedent";
|
|
415
|
-
import pRetry from "p-retry";
|
|
416
|
-
import z from "zod";
|
|
417
|
-
import "@deepagents/agent";
|
|
418
|
-
import {
|
|
419
|
-
ContextEngine,
|
|
420
|
-
InMemoryContextStore,
|
|
421
|
-
example,
|
|
422
|
-
fragment as fragment2,
|
|
423
|
-
guardrail,
|
|
424
|
-
hint,
|
|
425
|
-
persona,
|
|
426
|
-
policy,
|
|
427
|
-
structuredOutput,
|
|
428
|
-
user,
|
|
429
|
-
workflow
|
|
430
|
-
} from "@deepagents/context";
|
|
431
|
-
|
|
432
|
-
// packages/text2sql/src/lib/sql-scope-error.ts
|
|
433
|
-
function buildScopeParseErrorPayload(sql, parserDialect, error) {
|
|
434
|
-
const parserError = error instanceof Error ? error.message : String(error ?? "Unknown error");
|
|
435
|
-
return {
|
|
436
|
-
error: `SQL scope analysis failed before validation/execution: ${parserError}`,
|
|
437
|
-
error_type: "SQL_SCOPE_PARSE_ERROR",
|
|
438
|
-
suggestion: "Rewrite the query into simpler SQL that can be analyzed safely, or extend parser coverage for this dialect feature.",
|
|
439
|
-
sql_attempted: sql,
|
|
440
|
-
parser_dialect: parserDialect,
|
|
441
|
-
parser_error: parserError
|
|
442
|
-
};
|
|
443
|
-
}
|
|
444
|
-
|
|
445
|
-
// packages/text2sql/src/lib/agents/sql.agent.ts
|
|
446
|
-
var RETRY_TEMPERATURES = [0, 0.2, 0.3];
|
|
447
|
-
var SQL_AGENT_ROLE = "Expert SQL query generator.";
|
|
448
|
-
var SQL_AGENT_OBJECTIVE = "Generate precise SQL grounded in provided schema.";
|
|
449
|
-
var SQL_AGENT_POLICIES = [
|
|
450
|
-
fragment2(
|
|
451
|
-
"schema_mapping",
|
|
452
|
-
policy({
|
|
453
|
-
rule: "Translate natural language into precise SQL grounded in available schema entities."
|
|
454
|
-
}),
|
|
455
|
-
hint("Preserve schema spelling exactly, including typos in column names.")
|
|
456
|
-
),
|
|
457
|
-
fragment2(
|
|
458
|
-
"projection_minimality",
|
|
459
|
-
policy({
|
|
460
|
-
rule: "Return only columns requested by the question; do not add helper columns unless explicitly requested."
|
|
461
|
-
}),
|
|
462
|
-
policy({
|
|
463
|
-
rule: 'For requests of the form "X sorted/ordered by Y", project X only unless Y is explicitly requested as an output field.'
|
|
464
|
-
}),
|
|
465
|
-
policy({
|
|
466
|
-
rule: "Prefer selecting schema columns directly without derived expressions when direct selection answers the request."
|
|
467
|
-
}),
|
|
468
|
-
hint(
|
|
469
|
-
"Do not include ORDER BY, GROUP BY, or JOIN helper columns in SELECT output unless the question explicitly asks for them."
|
|
470
|
-
),
|
|
471
|
-
policy({
|
|
472
|
-
rule: "Use DISTINCT only when uniqueness is explicitly requested (for example distinct/unique/different/no duplicates)."
|
|
473
|
-
}),
|
|
474
|
-
hint(
|
|
475
|
-
'Do not infer DISTINCT from generic wording such as "some", plural nouns, or entity-set phrasing; for transactional/attendance-style tables, default to raw rows unless uniqueness is explicitly requested.'
|
|
476
|
-
)
|
|
477
|
-
),
|
|
478
|
-
fragment2(
|
|
479
|
-
"date_transform_safety",
|
|
480
|
-
policy({
|
|
481
|
-
rule: "Do not assume VARCHAR/TEXT values are parseable dates. Avoid date extraction functions on text columns by default."
|
|
482
|
-
}),
|
|
483
|
-
policy({
|
|
484
|
-
rule: "Use date-part extraction only when both conditions hold: the question explicitly asks for transformation and schema values require transformation to produce that unit."
|
|
485
|
-
}),
|
|
486
|
-
hint(
|
|
487
|
-
"Do not apply SUBSTR, STRFTIME, DATE_PART, YEAR, or similar extraction functions unless the question explicitly asks for transformation and schema values require it."
|
|
488
|
-
),
|
|
489
|
-
hint(
|
|
490
|
-
"If a column already represents the requested concept (for example a stored year-like value), use the column as-is."
|
|
491
|
-
)
|
|
492
|
-
),
|
|
493
|
-
fragment2(
|
|
494
|
-
"sql_minimality",
|
|
495
|
-
guardrail({
|
|
496
|
-
rule: "Never hallucinate tables or columns.",
|
|
497
|
-
reason: "Schema fidelity is required.",
|
|
498
|
-
action: "Use only available schema entities."
|
|
499
|
-
}),
|
|
500
|
-
guardrail({
|
|
501
|
-
rule: "Avoid unnecessary transformations and derived projections.",
|
|
502
|
-
reason: "Extra transformations frequently change semantics and reduce correctness.",
|
|
503
|
-
action: "Do not add date parsing, substring extraction, or derived columns unless explicitly required by the question or schema."
|
|
504
|
-
})
|
|
505
|
-
),
|
|
506
|
-
fragment2(
|
|
507
|
-
"preflight_checklist",
|
|
508
|
-
workflow({
|
|
509
|
-
task: "Final SQL preflight before returning output",
|
|
510
|
-
steps: [
|
|
511
|
-
"Verify selected columns match the question and remove unrequested helper projections.",
|
|
512
|
-
"If aggregate values are used only for ranking/filtering, keep them out of SELECT unless explicitly requested.",
|
|
513
|
-
"Prefer raw schema columns over derived expressions when raw columns already satisfy the request.",
|
|
514
|
-
"If a candidate query uses STRFTIME, SUBSTR, DATE_PART, YEAR, or similar extraction on text-like columns, remove that transformation unless explicitly required by the question.",
|
|
515
|
-
"Return only schema-grounded SQL using existing tables and columns."
|
|
516
|
-
]
|
|
517
|
-
})
|
|
518
|
-
),
|
|
519
|
-
fragment2(
|
|
520
|
-
"set_semantics",
|
|
521
|
-
policy({
|
|
522
|
-
rule: "For questions asking where both condition A and condition B hold over an attribute, compute the intersection of qualifying sets for that attribute."
|
|
523
|
-
}),
|
|
524
|
-
policy({
|
|
525
|
-
rule: "Do not force the same entity instance to satisfy both conditions unless the question explicitly requests the same person/row/entity."
|
|
526
|
-
}),
|
|
527
|
-
hint(
|
|
528
|
-
"Prefer INTERSECT (or logically equivalent set-based shape) over requiring the same physical row/entity to satisfy both conditions unless explicitly requested."
|
|
529
|
-
),
|
|
530
|
-
hint(
|
|
531
|
-
"When two conditions describe different row groups whose shared attribute is requested, build each group separately and intersect the attribute values."
|
|
532
|
-
),
|
|
533
|
-
hint(
|
|
534
|
-
"Do not collapse cross-group conditions into a single-row AND predicate when the intent is shared values across groups."
|
|
535
|
-
),
|
|
536
|
-
policy({
|
|
537
|
-
rule: "If two predicates on the same field cannot both be true for one row, do not combine them with AND; use set operations across separate filtered subsets when shared values are requested."
|
|
538
|
-
})
|
|
539
|
-
),
|
|
540
|
-
fragment2(
|
|
541
|
-
"predicate_column_alignment",
|
|
542
|
-
policy({
|
|
543
|
-
rule: "Match literal values to semantically compatible columns. Do not compare descriptive names to identifier columns."
|
|
544
|
-
}),
|
|
545
|
-
hint(
|
|
546
|
-
"When a filter value is a descriptive label (for example a department name), join through the lookup table and filter on its name/title column, not on *_id columns."
|
|
547
|
-
),
|
|
548
|
-
hint(
|
|
549
|
-
"When relation roles are explicit in wording (for example host/home/source/destination), prefer foreign keys with matching role qualifiers over generic similarly named columns."
|
|
550
|
-
),
|
|
551
|
-
policy({
|
|
552
|
-
rule: "When multiple foreign-key candidates exist, select the column whose qualifier best matches the relationship described in the question."
|
|
553
|
-
}),
|
|
554
|
-
policy({
|
|
555
|
-
rule: "For hosting/held semantics, prefer host_* relationship columns when available over generic *_id alternatives."
|
|
556
|
-
}),
|
|
557
|
-
hint(
|
|
558
|
-
'Interpret wording like "held/hosted a competition or event" as a hosting relationship and map to host_* foreign keys when present.'
|
|
559
|
-
),
|
|
560
|
-
policy({
|
|
561
|
-
rule: "Do not compare descriptive labels or names to *_id columns; join to the table containing the descriptive field and filter there."
|
|
562
|
-
}),
|
|
563
|
-
policy({
|
|
564
|
-
rule: "Keep numeric identifiers unquoted when used as numeric equality filters unless schema indicates text identifiers."
|
|
565
|
-
}),
|
|
566
|
-
policy({
|
|
567
|
-
rule: "When filtering by a descriptive label value and a related table exposes a corresponding *_name or title column, join to that table and filter on the descriptive column."
|
|
568
|
-
})
|
|
569
|
-
),
|
|
570
|
-
fragment2(
|
|
571
|
-
"ordering_semantics",
|
|
572
|
-
policy({
|
|
573
|
-
rule: "Respect explicit sort direction terms. If direction is not specified, use ascending order unless a superlative intent (most/least/highest/lowest) implies direction."
|
|
574
|
-
}),
|
|
575
|
-
policy({
|
|
576
|
-
rule: "When ranking categories by frequency, use COUNT for ordering but keep output focused on requested category fields unless counts are explicitly requested."
|
|
577
|
-
}),
|
|
578
|
-
policy({
|
|
579
|
-
rule: "Do not use DESC unless descending direction is explicit or a superlative intent requires descending ranking."
|
|
580
|
-
}),
|
|
581
|
-
policy({
|
|
582
|
-
rule: 'For "most common/frequent <attribute>" requests, return the attribute value(s) only; use counts only for ordering/filtering unless the question explicitly asks to return counts.'
|
|
583
|
-
}),
|
|
584
|
-
hint(
|
|
585
|
-
'Use DESC with LIMIT 1 for "most/highest/largest"; use ASC with LIMIT 1 for "least/lowest/smallest".'
|
|
586
|
-
)
|
|
587
|
-
),
|
|
588
|
-
fragment2(
|
|
589
|
-
"negative_membership_queries",
|
|
590
|
-
policy({
|
|
591
|
-
rule: "For requests asking entities that did not participate/host/appear in related records, prefer NOT IN or NOT EXISTS against the related foreign-key set."
|
|
592
|
-
}),
|
|
593
|
-
hint(
|
|
594
|
-
"Map role-bearing relationship columns carefully (for example host_* foreign keys for hosting relationships) instead of generic IDs when role wording is explicit."
|
|
595
|
-
),
|
|
596
|
-
hint(
|
|
597
|
-
'For "never had/never exceeded" conditions over history tables, exclude entities via NOT IN/NOT EXISTS against the disqualifying entity-id set (often built with GROUP BY/HAVING MAX(...)).'
|
|
598
|
-
)
|
|
599
|
-
),
|
|
600
|
-
fragment2(
|
|
601
|
-
"join_completeness",
|
|
602
|
-
policy({
|
|
603
|
-
rule: "Preserve entity-restricting joins implied by the question. Do not widen results by querying only a broader attribute table when a subset entity table is available."
|
|
604
|
-
}),
|
|
605
|
-
policy({
|
|
606
|
-
rule: "If an entity term in the question maps to a table, keep that table in query scope and join to attribute tables rather than dropping the entity table."
|
|
607
|
-
}),
|
|
608
|
-
hint(
|
|
609
|
-
"If the question targets a specific entity group, include that entity table and its join conditions even when selected columns come from a related table."
|
|
610
|
-
),
|
|
611
|
-
hint(
|
|
612
|
-
"When the question names an entity type and a relation table links to that entity via *_id, include the entity table in scope instead of counting only relation rows."
|
|
613
|
-
),
|
|
614
|
-
hint(
|
|
615
|
-
"Prefer INNER JOIN by default; use LEFT JOIN only when the question explicitly requests including unmatched rows or zero-related entities."
|
|
616
|
-
)
|
|
617
|
-
),
|
|
618
|
-
fragment2(
|
|
619
|
-
"aggregation_exactness",
|
|
620
|
-
policy({
|
|
621
|
-
rule: "Preserve requested aggregation semantics exactly: use COUNT(*) by default for total rows, use COUNT(DISTINCT ...) only when uniqueness is explicitly requested, and group by stable entity keys when computing per-entity aggregates."
|
|
622
|
-
}),
|
|
623
|
-
policy({
|
|
624
|
-
rule: "For questions asking which entity has lowest/highest average of a metric, compute AVG(metric) per entity (GROUP BY entity) and rank those aggregates."
|
|
625
|
-
}),
|
|
626
|
-
hint(
|
|
627
|
-
'For "how many <entities>" questions over relation records, default to COUNT(*) on qualifying rows unless explicit uniqueness language is present.'
|
|
628
|
-
)
|
|
629
|
-
),
|
|
630
|
-
fragment2(
|
|
631
|
-
"query_shape_examples",
|
|
632
|
-
example({
|
|
633
|
-
question: "List categories ordered by how many records belong to each category.",
|
|
634
|
-
answer: "SELECT category FROM records GROUP BY category ORDER BY COUNT(*)"
|
|
635
|
-
}),
|
|
636
|
-
example({
|
|
637
|
-
question: "Show labels shared by rows with metric > 100 and rows with metric < 10.",
|
|
638
|
-
answer: "SELECT label FROM records WHERE metric > 100 INTERSECT SELECT label FROM records WHERE metric < 10"
|
|
639
|
-
}),
|
|
640
|
-
example({
|
|
641
|
-
question: "List locations that have not hosted any event.",
|
|
642
|
-
answer: "SELECT location_name FROM locations WHERE location_id NOT IN (SELECT host_location_id FROM events)"
|
|
643
|
-
}),
|
|
644
|
-
example({
|
|
645
|
-
question: "List the most common category across records.",
|
|
646
|
-
answer: "SELECT category FROM records GROUP BY category ORDER BY COUNT(*) DESC LIMIT 1"
|
|
647
|
-
})
|
|
648
|
-
)
|
|
649
|
-
];
|
|
650
|
-
function extractSql(output) {
|
|
651
|
-
const match = output.match(/```sql\n?([\s\S]*?)```/);
|
|
652
|
-
return match ? match[1].trim() : output.trim();
|
|
653
|
-
}
|
|
654
|
-
async function toSql(options) {
|
|
655
|
-
const { maxRetries = 3 } = options;
|
|
656
|
-
return withRetry(
|
|
657
|
-
async (attemptNumber, errors, attempts) => {
|
|
658
|
-
const context = new ContextEngine({
|
|
659
|
-
store: new InMemoryContextStore(),
|
|
660
|
-
chatId: `sql-gen-${crypto.randomUUID()}`,
|
|
661
|
-
userId: "system"
|
|
662
|
-
});
|
|
663
|
-
context.set(
|
|
664
|
-
persona({
|
|
665
|
-
name: "Freya",
|
|
666
|
-
role: SQL_AGENT_ROLE,
|
|
667
|
-
objective: SQL_AGENT_OBJECTIVE
|
|
668
|
-
// role: `You are a data science expert that provides well-reasoned and detailed responses.`,
|
|
669
|
-
// objective: `Your task is to understand the schema and generate a valid SQL query to answer the question. You first think about the reasoning process as an internal monologue and then provide the user with the answer.`,
|
|
670
|
-
}),
|
|
671
|
-
...SQL_AGENT_POLICIES,
|
|
672
|
-
...options.fragments
|
|
673
|
-
);
|
|
674
|
-
if (errors.length) {
|
|
675
|
-
const lastError = errors.at(-1);
|
|
676
|
-
context.set(
|
|
677
|
-
user(dedent`
|
|
678
|
-
Answer the following question with the SQL code. Use the piece of evidence and base your answer on the database schema.
|
|
679
|
-
Given the question, the evidence and the database schema, return the SQL script that addresses the question.
|
|
680
|
-
|
|
681
|
-
Question: ${options.input}
|
|
682
|
-
`),
|
|
683
|
-
UnanswerableSQLError.isInstance(lastError) ? user(
|
|
684
|
-
`<retry_instruction>Your previous response marked the task as unanswerable. Re-evaluate using best-effort schema mapping. If the core intent is answerable with existing tables/columns, return SQL. Return error only when required core intent cannot be mapped without inventing schema elements.</retry_instruction>`
|
|
685
|
-
) : user(
|
|
686
|
-
`<validation_error>Your previous SQL query had the following error: ${lastError?.message}. Please fix the query.</validation_error>`
|
|
687
|
-
)
|
|
688
|
-
);
|
|
689
|
-
} else {
|
|
690
|
-
context.set(
|
|
691
|
-
user(dedent`
|
|
692
|
-
Answer the following question with the SQL code. Use the piece of evidence and base your answer on the database schema.
|
|
693
|
-
Given the question, the evidence and the database schema, return the SQL script that addresses the question.
|
|
694
|
-
|
|
695
|
-
Question: ${options.input}
|
|
696
|
-
`)
|
|
697
|
-
);
|
|
698
|
-
}
|
|
699
|
-
const temperature = RETRY_TEMPERATURES[attemptNumber - 1] ?? RETRY_TEMPERATURES[RETRY_TEMPERATURES.length - 1];
|
|
700
|
-
const baseModel = options.model ?? groq("openai/gpt-oss-20b");
|
|
701
|
-
const model = wrapLanguageModel({
|
|
702
|
-
model: baseModel,
|
|
703
|
-
middleware: defaultSettingsMiddleware({ settings: { temperature } })
|
|
704
|
-
});
|
|
705
|
-
const sqlOutput = structuredOutput({
|
|
706
|
-
model,
|
|
707
|
-
context,
|
|
708
|
-
schema: z.object({
|
|
709
|
-
result: z.union([
|
|
710
|
-
z.object({
|
|
711
|
-
sql: z.string().describe("The SQL query that answers the question"),
|
|
712
|
-
reasoning: z.string().describe("The reasoning steps taken to generate the SQL")
|
|
713
|
-
}),
|
|
714
|
-
z.object({
|
|
715
|
-
error: z.string().describe(
|
|
716
|
-
"Error message explaining why the question cannot be answered with the given schema"
|
|
717
|
-
)
|
|
718
|
-
})
|
|
719
|
-
])
|
|
720
|
-
})
|
|
721
|
-
});
|
|
722
|
-
const { result: output } = await sqlOutput.generate();
|
|
723
|
-
const finalizeSql = async (rawSql) => {
|
|
724
|
-
const extractedSql = extractSql(rawSql);
|
|
725
|
-
const errorsOrUndefined = errors.length ? errors.map(formatErrorMessage) : void 0;
|
|
726
|
-
if (extractedSql === "") {
|
|
727
|
-
return {
|
|
728
|
-
attempts,
|
|
729
|
-
sql: "",
|
|
730
|
-
errors: errorsOrUndefined
|
|
731
|
-
};
|
|
732
|
-
}
|
|
733
|
-
const sql = options.adapter.format(extractedSql);
|
|
734
|
-
if (rawSql.trimStart().startsWith("```") && extractedSql === rawSql.trim()) {
|
|
735
|
-
throw new SQLValidationError(
|
|
736
|
-
JSON.stringify(
|
|
737
|
-
buildScopeParseErrorPayload(
|
|
738
|
-
sql,
|
|
739
|
-
options.adapter.formatterLanguage,
|
|
740
|
-
new Error(
|
|
741
|
-
"SQL response was fenced markdown, not executable SQL."
|
|
742
|
-
)
|
|
743
|
-
)
|
|
744
|
-
)
|
|
745
|
-
);
|
|
746
|
-
}
|
|
747
|
-
const validationError = await options.adapter.validate(sql);
|
|
748
|
-
if (validationError) {
|
|
749
|
-
throw new SQLValidationError(validationError);
|
|
750
|
-
}
|
|
751
|
-
return {
|
|
752
|
-
attempts,
|
|
753
|
-
sql,
|
|
754
|
-
errors: errorsOrUndefined
|
|
755
|
-
};
|
|
756
|
-
};
|
|
757
|
-
if ("error" in output) {
|
|
758
|
-
context.set(
|
|
759
|
-
user(
|
|
760
|
-
"<best_effort_fallback>Do not return unanswerable. Produce the best valid SQL query that answers the core intent using only available schema entities.</best_effort_fallback>"
|
|
761
|
-
)
|
|
762
|
-
);
|
|
763
|
-
const forcedSqlOutput = structuredOutput({
|
|
764
|
-
model,
|
|
765
|
-
context,
|
|
766
|
-
schema: z.object({
|
|
767
|
-
sql: z.string().describe(
|
|
768
|
-
"Best-effort SQL query that answers the core intent using only available schema entities."
|
|
769
|
-
),
|
|
770
|
-
reasoning: z.string().describe("Reasoning steps for best-effort schema mapping.")
|
|
771
|
-
})
|
|
772
|
-
});
|
|
773
|
-
try {
|
|
774
|
-
const forced = await forcedSqlOutput.generate();
|
|
775
|
-
return await finalizeSql(forced.sql);
|
|
776
|
-
} catch (error) {
|
|
777
|
-
if (SQLValidationError.isInstance(error) || APICallError.isInstance(error) || JSONParseError.isInstance(error) || TypeValidationError.isInstance(error) || NoObjectGeneratedError.isInstance(error) || NoOutputGeneratedError.isInstance(error) || NoContentGeneratedError.isInstance(error)) {
|
|
778
|
-
throw error;
|
|
779
|
-
}
|
|
780
|
-
throw new UnanswerableSQLError(output.error);
|
|
781
|
-
}
|
|
782
|
-
}
|
|
783
|
-
return await finalizeSql(output.sql);
|
|
784
|
-
},
|
|
785
|
-
{ retries: maxRetries - 1 }
|
|
786
|
-
);
|
|
787
|
-
}
|
|
788
|
-
function formatErrorMessage(error) {
|
|
789
|
-
if (APICallError.isInstance(error)) {
|
|
790
|
-
if (error.message.startsWith("Failed to validate JSON")) {
|
|
791
|
-
return `Schema validation failed: ${error.message}`;
|
|
792
|
-
}
|
|
793
|
-
return error.message;
|
|
794
|
-
}
|
|
795
|
-
if (SQLValidationError.isInstance(error)) {
|
|
796
|
-
return `SQL Validation Error: ${error.message}`;
|
|
797
|
-
}
|
|
798
|
-
return error.message;
|
|
799
|
-
}
|
|
800
|
-
function isModelUnavailableError(error) {
|
|
801
|
-
if (!APICallError.isInstance(error)) {
|
|
802
|
-
return false;
|
|
803
|
-
}
|
|
804
|
-
const message = error.message.toLowerCase();
|
|
805
|
-
const responseBody = (error.responseBody ?? "").toLowerCase();
|
|
806
|
-
const is404ModelError = error.statusCode === 404 && (message.includes("model") || responseBody.includes("model_not_found"));
|
|
807
|
-
const errorCode = typeof error.data === "object" && error.data !== null && "error" in error.data && typeof error.data.error === "object" && error.data.error !== null && "code" in error.data.error && typeof error.data.error.code === "string" ? error.data.error.code.toLowerCase() : void 0;
|
|
808
|
-
return is404ModelError || errorCode === "model_not_found" || responseBody.includes('"code":"model_not_found"') || message.includes("model") && message.includes("does not exist or you do not have access to it");
|
|
809
|
-
}
|
|
810
|
-
async function withRetry(computation, options = { retries: 3 }) {
|
|
811
|
-
const errors = [];
|
|
812
|
-
let attempts = 0;
|
|
813
|
-
return pRetry(
|
|
814
|
-
(attemptNumber) => {
|
|
815
|
-
return computation(attemptNumber, errors, ++attempts);
|
|
816
|
-
},
|
|
817
|
-
{
|
|
818
|
-
retries: options.retries,
|
|
819
|
-
shouldRetry: (context) => {
|
|
820
|
-
if (UnanswerableSQLError.isInstance(context.error)) {
|
|
821
|
-
return false;
|
|
822
|
-
}
|
|
823
|
-
if (isModelUnavailableError(context.error)) {
|
|
824
|
-
return false;
|
|
825
|
-
}
|
|
826
|
-
if (SQLValidationError.isInstance(context.error)) {
|
|
827
|
-
return true;
|
|
828
|
-
}
|
|
829
|
-
console.log({
|
|
830
|
-
NoObjectGeneratedError: NoObjectGeneratedError.isInstance(
|
|
831
|
-
context.error
|
|
832
|
-
),
|
|
833
|
-
NoOutputGeneratedError: NoOutputGeneratedError.isInstance(
|
|
834
|
-
context.error
|
|
835
|
-
),
|
|
836
|
-
APICallError: APICallError.isInstance(context.error),
|
|
837
|
-
JSONParseError: JSONParseError.isInstance(context.error),
|
|
838
|
-
TypeValidationError: TypeValidationError.isInstance(context.error),
|
|
839
|
-
NoContentGeneratedError: NoContentGeneratedError.isInstance(
|
|
840
|
-
context.error
|
|
841
|
-
)
|
|
842
|
-
});
|
|
843
|
-
return APICallError.isInstance(context.error) || JSONParseError.isInstance(context.error) || TypeValidationError.isInstance(context.error) || NoObjectGeneratedError.isInstance(context.error) || NoOutputGeneratedError.isInstance(context.error) || NoContentGeneratedError.isInstance(context.error);
|
|
844
|
-
},
|
|
845
|
-
onFailedAttempt(context) {
|
|
846
|
-
errors.push(context.error);
|
|
847
|
-
}
|
|
848
|
-
}
|
|
849
|
-
);
|
|
850
|
-
}
|
|
851
|
-
|
|
852
354
|
// packages/text2sql/src/lib/sql.ts
|
|
853
355
|
var text2SqlValidationMarker = /* @__PURE__ */ Symbol("Text2SqlValidationError");
|
|
854
356
|
var text2SqlUnknownAdapterMarker = /* @__PURE__ */ Symbol("Text2SqlUnknownAdapterError");
|
|
@@ -910,8 +412,7 @@ var Text2Sql = class {
|
|
|
910
412
|
this.#config = config;
|
|
911
413
|
this.#indexer = new AdapterIndexer({
|
|
912
414
|
adapters: config.adapters,
|
|
913
|
-
cache: config.cache
|
|
914
|
-
lock: config.lock
|
|
415
|
+
cache: config.cache
|
|
915
416
|
});
|
|
916
417
|
}
|
|
917
418
|
#requireAdapter(name) {
|
|
@@ -921,13 +422,6 @@ var Text2Sql = class {
|
|
|
921
422
|
}
|
|
922
423
|
return adapter;
|
|
923
424
|
}
|
|
924
|
-
#requireModel(op) {
|
|
925
|
-
const model = this.#config.model;
|
|
926
|
-
if (!model) {
|
|
927
|
-
throw new Error(`Text2Sql.${op}() requires a model in Text2SqlConfig`);
|
|
928
|
-
}
|
|
929
|
-
return model;
|
|
930
|
-
}
|
|
931
425
|
adapterNames() {
|
|
932
426
|
return Object.keys(this.#config.adapters);
|
|
933
427
|
}
|
|
@@ -969,8 +463,10 @@ var Text2Sql = class {
|
|
|
969
463
|
if (!Array.isArray(result)) {
|
|
970
464
|
throw new Error("adapter.execute must return an array of rows");
|
|
971
465
|
}
|
|
972
|
-
const
|
|
973
|
-
|
|
466
|
+
const rows = result;
|
|
467
|
+
const [first] = rows;
|
|
468
|
+
const columns = typeof first === "object" && first !== null ? Object.keys(first) : [];
|
|
469
|
+
return { rows, columns };
|
|
974
470
|
}
|
|
975
471
|
async #validateWith(adapter, sql) {
|
|
976
472
|
const formatted = adapter.format(sql);
|
|
@@ -984,18 +480,6 @@ var Text2Sql = class {
|
|
|
984
480
|
onProgress: options.onProgress
|
|
985
481
|
});
|
|
986
482
|
}
|
|
987
|
-
async toSql(input, adapterName) {
|
|
988
|
-
const adapter = this.#requireAdapter(adapterName);
|
|
989
|
-
const model = this.#requireModel("toSql");
|
|
990
|
-
const fragments = await this.#indexer.indexAdapter(adapterName);
|
|
991
|
-
const result = await toSql({
|
|
992
|
-
input,
|
|
993
|
-
adapter,
|
|
994
|
-
fragments,
|
|
995
|
-
model
|
|
996
|
-
});
|
|
997
|
-
return result.sql;
|
|
998
|
-
}
|
|
999
483
|
async toPairs(adapterName, factory) {
|
|
1000
484
|
const adapter = this.#requireAdapter(adapterName);
|
|
1001
485
|
const producer = factory(adapter);
|
|
@@ -1004,14 +488,18 @@ var Text2Sql = class {
|
|
|
1004
488
|
};
|
|
1005
489
|
|
|
1006
490
|
// packages/text2sql/src/bin/command.ts
|
|
1007
|
-
import * as
|
|
491
|
+
import * as path2 from "node:path";
|
|
492
|
+
import z2 from "zod";
|
|
493
|
+
var cliWord = z2.union([z2.string(), z2.number()]).transform(String);
|
|
494
|
+
var cliOptions = z2.record(z2.string(), z2.unknown());
|
|
495
|
+
var queryArgs = z2.tuple([cliWord, z2.array(cliWord)]);
|
|
1008
496
|
var OUT_DIR_OPTION = {
|
|
1009
497
|
flag: "--out-dir <path>",
|
|
1010
498
|
description: "Output directory (default: $TEXT2SQL_OUT_DIR or ./sql)"
|
|
1011
499
|
};
|
|
1012
500
|
function resolveOutputDir(ctx, options) {
|
|
1013
501
|
const explicit = typeof options.outDir === "string" ? options.outDir : void 0;
|
|
1014
|
-
return
|
|
502
|
+
return path2.resolve(ctx.cwd, explicit ?? ctx.env.TEXT2SQL_OUT_DIR ?? "./sql");
|
|
1015
503
|
}
|
|
1016
504
|
var CommandError = class extends Error {
|
|
1017
505
|
command;
|
|
@@ -1047,9 +535,9 @@ var SqlCommand = class {
|
|
|
1047
535
|
var SqlQueryCommand = class extends SqlCommand {
|
|
1048
536
|
args = "<db> <...sql>";
|
|
1049
537
|
async execute(ctx, args, options) {
|
|
1050
|
-
const [db, sqlParts] = args;
|
|
538
|
+
const [db, sqlParts] = queryArgs.parse(args);
|
|
1051
539
|
const name = this.resolveAdapterName(ctx.text2Sql, db);
|
|
1052
|
-
const sql =
|
|
540
|
+
const sql = sqlParts.join(" ").trim();
|
|
1053
541
|
if (!sql) this.fail("no query provided");
|
|
1054
542
|
try {
|
|
1055
543
|
return await this.runQuery(ctx, name, sql, options);
|
|
@@ -1064,14 +552,15 @@ var SqlQueryCommand = class extends SqlCommand {
|
|
|
1064
552
|
// packages/text2sql/src/bin/commands/index.ts
|
|
1065
553
|
import { constants, createWriteStream } from "node:fs";
|
|
1066
554
|
import {
|
|
1067
|
-
mkdir as
|
|
555
|
+
mkdir as mkdir2,
|
|
1068
556
|
open,
|
|
1069
557
|
stat,
|
|
1070
558
|
writeFile as writeFile2
|
|
1071
559
|
} from "node:fs/promises";
|
|
1072
|
-
import * as
|
|
560
|
+
import * as path3 from "node:path";
|
|
1073
561
|
import { finished } from "node:stream/promises";
|
|
1074
562
|
import { v7 } from "uuid";
|
|
563
|
+
import z3 from "zod";
|
|
1075
564
|
|
|
1076
565
|
// packages/text2sql/src/bin/commands/indexing-formatter.ts
|
|
1077
566
|
var REDUNDANT_MESSAGE_TYPES = /* @__PURE__ */ new Set([
|
|
@@ -1092,6 +581,7 @@ function formatPretty(event) {
|
|
|
1092
581
|
}
|
|
1093
582
|
|
|
1094
583
|
// packages/text2sql/src/bin/commands/index.ts
|
|
584
|
+
var indexArgs = z3.tuple([z3.array(cliWord)]);
|
|
1095
585
|
var IndexCommand = class extends SqlCommand {
|
|
1096
586
|
name = "index";
|
|
1097
587
|
description = "Index adapter schemas and write context artifacts";
|
|
@@ -1107,19 +597,19 @@ var IndexCommand = class extends SqlCommand {
|
|
|
1107
597
|
OUT_DIR_OPTION
|
|
1108
598
|
];
|
|
1109
599
|
async execute(ctx, args, options) {
|
|
1110
|
-
const adapterNames = args
|
|
600
|
+
const [adapterNames] = indexArgs.parse(args);
|
|
1111
601
|
const requested = options.all ? [] : adapterNames;
|
|
1112
602
|
const names = this.resolveNames(ctx.text2Sql, requested);
|
|
1113
603
|
const verbose = this.resolveVerbose(options.verbose);
|
|
1114
604
|
const outputDir = resolveOutputDir(ctx, options);
|
|
1115
605
|
const id = v7();
|
|
1116
|
-
const fragmentsPath =
|
|
606
|
+
const fragmentsPath = path3.join(outputDir, `index-${id}.json`);
|
|
1117
607
|
const explicitEventsPath = resolveExplicitEventsPath(ctx);
|
|
1118
|
-
const eventsPath = explicitEventsPath ??
|
|
608
|
+
const eventsPath = explicitEventsPath ?? path3.join(outputDir, `index-${id}.events.ndjson`);
|
|
1119
609
|
try {
|
|
1120
|
-
await
|
|
610
|
+
await mkdir2(outputDir, { recursive: true });
|
|
1121
611
|
if (explicitEventsPath) {
|
|
1122
|
-
await
|
|
612
|
+
await mkdir2(path3.dirname(explicitEventsPath), { recursive: true });
|
|
1123
613
|
}
|
|
1124
614
|
const eventsStream = await openEventsWriter(
|
|
1125
615
|
eventsPath,
|
|
@@ -1179,7 +669,7 @@ function createProgressHandler(eventsStream, verbose, stderr) {
|
|
|
1179
669
|
}
|
|
1180
670
|
function resolveExplicitEventsPath(ctx) {
|
|
1181
671
|
const explicit = ctx.env.TEXT2SQL_INDEX_EVENTS_PATH?.trim();
|
|
1182
|
-
return explicit ?
|
|
672
|
+
return explicit ? path3.resolve(ctx.cwd, explicit) : null;
|
|
1183
673
|
}
|
|
1184
674
|
async function openEventsWriter(eventsPath, explicitEventsPath, stderr) {
|
|
1185
675
|
const probe = explicitEventsPath ? await probeFifoWriter(explicitEventsPath, stderr) : null;
|
|
@@ -1203,14 +693,14 @@ async function probeFifoWriter(eventsPath, stderr) {
|
|
|
1203
693
|
try {
|
|
1204
694
|
stats = await stat(eventsPath);
|
|
1205
695
|
} catch (err) {
|
|
1206
|
-
if (err
|
|
696
|
+
if (hasErrorCode(err, "ENOENT")) return null;
|
|
1207
697
|
throw err;
|
|
1208
698
|
}
|
|
1209
699
|
if (!stats.isFIFO()) return null;
|
|
1210
700
|
try {
|
|
1211
701
|
return await open(eventsPath, constants.O_WRONLY | constants.O_NONBLOCK);
|
|
1212
702
|
} catch (err) {
|
|
1213
|
-
if (err
|
|
703
|
+
if (hasErrorCode(err, "ENXIO")) {
|
|
1214
704
|
stderr.write(`sql index: waiting for reader on FIFO ${eventsPath}...
|
|
1215
705
|
`);
|
|
1216
706
|
return null;
|
|
@@ -1218,6 +708,9 @@ async function probeFifoWriter(eventsPath, stderr) {
|
|
|
1218
708
|
throw err;
|
|
1219
709
|
}
|
|
1220
710
|
}
|
|
711
|
+
function hasErrorCode(error, code) {
|
|
712
|
+
return error instanceof Error && "code" in error && error.code === code;
|
|
713
|
+
}
|
|
1221
714
|
function waitForOpen(stream) {
|
|
1222
715
|
return new Promise((resolve4, reject) => {
|
|
1223
716
|
if (!stream.pending) {
|
|
@@ -1241,8 +734,8 @@ function dedupe(values) {
|
|
|
1241
734
|
}
|
|
1242
735
|
|
|
1243
736
|
// packages/text2sql/src/bin/commands/run.ts
|
|
1244
|
-
import { mkdir as
|
|
1245
|
-
import * as
|
|
737
|
+
import { mkdir as mkdir3, writeFile as writeFile3 } from "node:fs/promises";
|
|
738
|
+
import * as path4 from "node:path";
|
|
1246
739
|
import { v7 as v72 } from "uuid";
|
|
1247
740
|
var RunCommand = class extends SqlQueryCommand {
|
|
1248
741
|
name = "run";
|
|
@@ -1253,9 +746,9 @@ var RunCommand = class extends SqlQueryCommand {
|
|
|
1253
746
|
async runQuery(ctx, name, sql, options) {
|
|
1254
747
|
const outputDir = resolveOutputDir(ctx, options);
|
|
1255
748
|
const { rows, columns } = await ctx.text2Sql.run(name, sql);
|
|
1256
|
-
const outPath =
|
|
749
|
+
const outPath = path4.join(outputDir, `${v72()}.json`);
|
|
1257
750
|
try {
|
|
1258
|
-
await
|
|
751
|
+
await mkdir3(outputDir, { recursive: true });
|
|
1259
752
|
await writeFile3(outPath, JSON.stringify(rows, null, 2));
|
|
1260
753
|
} catch (error) {
|
|
1261
754
|
this.fail(errorMessage2(error));
|
|
@@ -1301,15 +794,12 @@ async function loadAdapters() {
|
|
|
1301
794
|
);
|
|
1302
795
|
}
|
|
1303
796
|
const specifier = target.startsWith(".") || target.startsWith("/") ? pathToFileURL(resolve3(target)).href : target;
|
|
1304
|
-
|
|
1305
|
-
try {
|
|
1306
|
-
mod = await import(specifier);
|
|
1307
|
-
} catch (cause) {
|
|
797
|
+
const mod = await import(specifier).catch((cause) => {
|
|
1308
798
|
throw new Error(
|
|
1309
799
|
`TEXT2SQL_ADAPTERS=${target}: failed to import module - ${errorMessage2(cause)}`
|
|
1310
800
|
);
|
|
1311
|
-
}
|
|
1312
|
-
const exported = mod.default;
|
|
801
|
+
});
|
|
802
|
+
const exported = typeof mod === "object" && mod !== null && "default" in mod ? mod.default : void 0;
|
|
1313
803
|
if (!exported || typeof exported !== "object" || Array.isArray(exported)) {
|
|
1314
804
|
throw new Error(
|
|
1315
805
|
`TEXT2SQL_ADAPTERS=${target}: default export must be a Record<string, Adapter> (got ${describe(exported)}).`
|
|
@@ -1321,24 +811,24 @@ async function loadAdapters() {
|
|
|
1321
811
|
`TEXT2SQL_ADAPTERS=${target}: default export is an empty object - declare at least one adapter.`
|
|
1322
812
|
);
|
|
1323
813
|
}
|
|
814
|
+
const adapters = {};
|
|
1324
815
|
for (const [name, value] of entries) {
|
|
1325
816
|
if (!isAdapterShape(value)) {
|
|
1326
817
|
throw new Error(
|
|
1327
818
|
`TEXT2SQL_ADAPTERS=${target}: adapter "${name}" is missing one of the required methods (format, validate, execute).`
|
|
1328
819
|
);
|
|
1329
820
|
}
|
|
821
|
+
adapters[name] = value;
|
|
1330
822
|
}
|
|
1331
823
|
try {
|
|
1332
824
|
validateAdapterNames(entries.map(([name]) => name));
|
|
1333
825
|
} catch (cause) {
|
|
1334
826
|
throw new Error(`TEXT2SQL_ADAPTERS=${target}: ${errorMessage2(cause)}`);
|
|
1335
827
|
}
|
|
1336
|
-
return
|
|
828
|
+
return adapters;
|
|
1337
829
|
}
|
|
1338
830
|
function isAdapterShape(value) {
|
|
1339
|
-
|
|
1340
|
-
const v = value;
|
|
1341
|
-
return typeof v.format === "function" && typeof v.validate === "function" && typeof v.execute === "function";
|
|
831
|
+
return typeof value === "object" && value !== null && "format" in value && typeof value.format === "function" && "validate" in value && typeof value.validate === "function" && "execute" in value && typeof value.execute === "function";
|
|
1342
832
|
}
|
|
1343
833
|
function describe(value) {
|
|
1344
834
|
if (value === null) return "null";
|
|
@@ -1387,7 +877,7 @@ function registerCommand(cli2, command) {
|
|
|
1387
877
|
cliCommand.option(option.flag, option.description);
|
|
1388
878
|
}
|
|
1389
879
|
cliCommand.action(async (...callArgs) => {
|
|
1390
|
-
const options = callArgs
|
|
880
|
+
const options = cliOptions.parse(callArgs.at(-1));
|
|
1391
881
|
const positional = callArgs.slice(0, -1);
|
|
1392
882
|
return runCommand(command, positional, options);
|
|
1393
883
|
});
|
|
@@ -1400,19 +890,12 @@ function resolveIndexCache() {
|
|
|
1400
890
|
}
|
|
1401
891
|
return new FileIndexCache({ dir, namespace });
|
|
1402
892
|
}
|
|
1403
|
-
function resolveIndexLock() {
|
|
1404
|
-
return new FileIndexLock({
|
|
1405
|
-
dir: process.env.TEXT2SQL_INDEX_LOCK_DIR ?? process.env.TEXT2SQL_INDEX_CACHE_DIR,
|
|
1406
|
-
namespace: process.env.TEXT2SQL_INDEX_VERSION
|
|
1407
|
-
});
|
|
1408
|
-
}
|
|
1409
893
|
async function runCommand(command, positional, options) {
|
|
1410
894
|
let ctx;
|
|
1411
895
|
try {
|
|
1412
896
|
const text2Sql = new Text2Sql({
|
|
1413
897
|
adapters: await loadAdapters(),
|
|
1414
|
-
cache: resolveIndexCache()
|
|
1415
|
-
lock: resolveIndexLock()
|
|
898
|
+
cache: resolveIndexCache()
|
|
1416
899
|
});
|
|
1417
900
|
ctx = {
|
|
1418
901
|
text2Sql,
|