@deepagents/text2sql 7.2.0 → 7.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +80 -28
- package/dist/bin/sql.js +22 -15
- package/dist/finetune/convert-to-gguf.d.ts +0 -16
- package/dist/finetune/convert-to-gguf.d.ts.map +1 -1
- package/dist/index.js +52 -35
- package/dist/index.js.map +2 -2
- package/dist/lib/adapters/adapter.d.ts.map +1 -1
- package/dist/lib/adapters/bigquery/index.js +2 -6
- package/dist/lib/adapters/bigquery/index.js.map +2 -2
- package/dist/lib/adapters/clickhouse/index.js +4 -8
- package/dist/lib/adapters/clickhouse/index.js.map +3 -3
- package/dist/lib/adapters/duckdb/index.js +6 -10
- package/dist/lib/adapters/duckdb/index.js.map +3 -3
- package/dist/lib/adapters/mysql/index.js +2 -6
- package/dist/lib/adapters/mysql/index.js.map +2 -2
- package/dist/lib/adapters/postgres/index.js +4 -8
- package/dist/lib/adapters/postgres/index.js.map +3 -3
- package/dist/lib/adapters/posthog/index.js +2 -6
- package/dist/lib/adapters/posthog/index.js.map +2 -2
- package/dist/lib/adapters/spreadsheet/index.js +7 -13
- package/dist/lib/adapters/spreadsheet/index.js.map +3 -3
- package/dist/lib/adapters/spreadsheet/spreadsheet.d.ts.map +1 -1
- package/dist/lib/adapters/sqlite/index.js +5 -11
- package/dist/lib/adapters/sqlite/index.js.map +3 -3
- package/dist/lib/adapters/sqlite/table.sqlite.grounding.d.ts.map +1 -1
- package/dist/lib/adapters/sqlserver/index.js +2 -6
- package/dist/lib/adapters/sqlserver/index.js.map +2 -2
- package/dist/lib/checkpoint.d.ts +9 -0
- package/dist/lib/checkpoint.d.ts.map +1 -1
- package/dist/lib/file-cache.d.ts +12 -2
- package/dist/lib/file-cache.d.ts.map +1 -1
- package/dist/lib/fs/sqlite/ddl.sqlite-fs.d.ts +1 -1
- package/dist/lib/fs/sqlite/ddl.sqlite-fs.d.ts.map +1 -1
- package/dist/lib/fs/sqlite/sqlite-fs.d.ts.map +1 -1
- package/dist/lib/index-cache.d.ts +4 -1
- package/dist/lib/index-cache.d.ts.map +1 -1
- package/package.json +15 -9
package/README.md
CHANGED
|
@@ -4,12 +4,12 @@ AI-powered natural language to SQL. Ask questions in plain English, get executab
|
|
|
4
4
|
|
|
5
5
|
## Features
|
|
6
6
|
|
|
7
|
-
- **Natural
|
|
8
|
-
- **
|
|
9
|
-
- **Schema
|
|
10
|
-
- **Domain
|
|
7
|
+
- **Natural language to SQL** - Convert questions to validated, executable queries
|
|
8
|
+
- **Database adapters** - PostgreSQL, SQLite, DuckDB, SQL Server, MySQL/MariaDB, BigQuery, ClickHouse, and PostHog HogQL adapters
|
|
9
|
+
- **Schema introspection** - Automatic introspection of tables, relationships, indexes, and constraints
|
|
10
|
+
- **Domain knowledge** - Inject business terms, guardrails, and query patterns via fragments
|
|
11
11
|
- **Conversational** - Multi-turn conversations with context persistence
|
|
12
|
-
- **Safe by
|
|
12
|
+
- **Safe by default** - Read-only queries, validation, and configurable guardrails
|
|
13
13
|
|
|
14
14
|
## Installation
|
|
15
15
|
|
|
@@ -29,9 +29,24 @@ npm install @clickhouse/client # ClickHouse, or use your preferred client
|
|
|
29
29
|
# PostHog uses native fetch and needs no additional client package
|
|
30
30
|
```
|
|
31
31
|
|
|
32
|
-
|
|
32
|
+
The spreadsheet adapter requires `xlsx` for Excel, CSV, and TSV files. It is
|
|
33
|
+
an optional peer dependency; install it only if you use
|
|
34
|
+
`@deepagents/text2sql/spreadsheet`. With npm 12, add this to your project's
|
|
35
|
+
`.npmrc` to allow directly declared URL dependencies:
|
|
33
36
|
|
|
34
|
-
|
|
37
|
+
```ini
|
|
38
|
+
allow-remote=root
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
Then install SheetJS `xlsx` 0.20.3 from its CDN:
|
|
42
|
+
|
|
43
|
+
```bash
|
|
44
|
+
npm install https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
Requires Node.js LTS.
|
|
48
|
+
|
|
49
|
+
## Quick start
|
|
35
50
|
|
|
36
51
|
```typescript
|
|
37
52
|
import { groq } from '@ai-sdk/groq';
|
|
@@ -96,11 +111,11 @@ maps dynamically, use `isValidAdapterName(name)` to check one key or
|
|
|
96
111
|
`validateAdapterNames(names)` to fail fast before constructing `Text2Sql` or
|
|
97
112
|
the sandbox-side adapter module used by the `sql` CLI.
|
|
98
113
|
|
|
99
|
-
## AI
|
|
114
|
+
## AI model providers
|
|
100
115
|
|
|
101
116
|
Text2SQL works with any model provider supported by the [Vercel AI SDK](https://sdk.vercel.ai/docs), including OpenAI, Anthropic, Google, Groq, and more.
|
|
102
117
|
|
|
103
|
-
## Building a
|
|
118
|
+
## Building a conversational agent
|
|
104
119
|
|
|
105
120
|
`Text2Sql` owns SQL generation and schema indexing, but you own the chat loop.
|
|
106
121
|
Build the agent by composing a `ContextEngine`, a sandbox, and `agent` +
|
|
@@ -144,6 +159,7 @@ const backend = await createDockerSandbox({
|
|
|
144
159
|
],
|
|
145
160
|
env: {
|
|
146
161
|
TEXT2SQL_ADAPTERS: '/workspace/text2sql-adapters.ts',
|
|
162
|
+
TEXT2SQL_OUT_DIR: '/tmp/text2sql',
|
|
147
163
|
},
|
|
148
164
|
});
|
|
149
165
|
const sandbox = await createBashTool({
|
|
@@ -179,7 +195,12 @@ for await (const chunk of stream) {
|
|
|
179
195
|
|
|
180
196
|
The `/workspace/text2sql-adapters.ts` module must exist in the sandbox and
|
|
181
197
|
default-export your adapter map. Mount your project into `/workspace` (as above)
|
|
182
|
-
or upload
|
|
198
|
+
or upload or write that module before calling `sql index`. With a remote Docker
|
|
199
|
+
daemon, the bind mount's `hostPath` must exist on the daemon's host.
|
|
200
|
+
|
|
201
|
+
`TEXT2SQL_OUT_DIR` keeps schema and query artifacts in the sandbox's writable
|
|
202
|
+
`/tmp/text2sql` directory. The default `./sql` would be inside the read-only
|
|
203
|
+
`/workspace` mount.
|
|
183
204
|
|
|
184
205
|
`instructions()` returns the SQL-flavored system fragments (policies, workflows,
|
|
185
206
|
clarifications, style guides) — spread them into `context.set()` alongside the
|
|
@@ -197,7 +218,7 @@ turns, the reminder is persisted as a synthetic user message for replay and
|
|
|
197
218
|
prompt caching; bare `createPrepareStep()` integrations and
|
|
198
219
|
`agent.generate()` own persistence of their generated assistant history.
|
|
199
220
|
|
|
200
|
-
## Advanced: SQL CLI in
|
|
221
|
+
## Advanced: SQL CLI in sandboxes
|
|
201
222
|
|
|
202
223
|
`sql validate <db> "..."` and `sql run <db> "..."` are real commands from the
|
|
203
224
|
`@deepagents/text2sql` package. `<db>` is the configured adapter name. With a
|
|
@@ -228,11 +249,13 @@ changes). With neither set, every `sql index` introspects fresh. See
|
|
|
228
249
|
[Schema index caching](#schema-index-caching) for
|
|
229
250
|
the underlying injectable primitives.
|
|
230
251
|
|
|
231
|
-
For in-process or virtual-sandbox usage
|
|
232
|
-
|
|
252
|
+
For in-process or virtual-sandbox usage, install `just-bash` and wrap an existing
|
|
253
|
+
`Text2Sql` instance as a custom command:
|
|
233
254
|
|
|
234
255
|
```typescript
|
|
235
|
-
import { InMemoryFs
|
|
256
|
+
import { InMemoryFs } from 'just-bash';
|
|
257
|
+
|
|
258
|
+
import { createVirtualSandbox } from '@deepagents/context';
|
|
236
259
|
import {
|
|
237
260
|
type CreateSqlCommandOptions,
|
|
238
261
|
type CreateSqlCommandResult,
|
|
@@ -396,6 +419,26 @@ for await (const chunk of followUp) {
|
|
|
396
419
|
}
|
|
397
420
|
```
|
|
398
421
|
|
|
422
|
+
## Checkpoint files
|
|
423
|
+
|
|
424
|
+
`Checkpoint` from `@deepagents/text2sql` saves completed computations so a
|
|
425
|
+
pipeline can resume without repeating them. Load a file with
|
|
426
|
+
`Checkpoint.load({ path, configHash, clock: globalThis })`, then use `run()` for
|
|
427
|
+
one computation or `each()` for a sequence of inputs. Both require a codec to
|
|
428
|
+
encode outputs and validate them on resume; `jsonCodec(schema)` handles plain
|
|
429
|
+
JSON values. The optional `configHash` invalidates saved results when both the
|
|
430
|
+
file and the caller supply nonempty hashes that differ.
|
|
431
|
+
|
|
432
|
+
`clock` is required and controls the delay between file replacement retries
|
|
433
|
+
when a file is busy. Pass `globalThis` to use Node.js timers.
|
|
434
|
+
|
|
435
|
+
Writes from one `Checkpoint` instance run in order. Each write captures the
|
|
436
|
+
latest state when it starts, so an earlier snapshot cannot overwrite a later
|
|
437
|
+
one. The checkpoint uses `durableWrite` to flush the file before replacing it and
|
|
438
|
+
sync the parent directory where supported. Await `run()`, `each()`, or
|
|
439
|
+
`point().through()` before exiting. Separate instances or processes do not
|
|
440
|
+
coordinate writes to the same file.
|
|
441
|
+
|
|
399
442
|
## Schema index caching
|
|
400
443
|
|
|
401
444
|
Schema introspection is the expensive part of indexing. `Text2Sql` and
|
|
@@ -420,13 +463,20 @@ import { FileIndexCache, Text2Sql } from '@deepagents/text2sql';
|
|
|
420
463
|
|
|
421
464
|
const text2sql = new Text2Sql({
|
|
422
465
|
adapters: { main: adapter },
|
|
423
|
-
cache: new FileIndexCache({
|
|
466
|
+
cache: new FileIndexCache({
|
|
467
|
+
clock: globalThis,
|
|
468
|
+
dir: '/var/cache/text2sql',
|
|
469
|
+
namespace: 'v1',
|
|
470
|
+
}),
|
|
424
471
|
});
|
|
425
472
|
```
|
|
426
473
|
|
|
427
|
-
`FileIndexCache` writes
|
|
428
|
-
|
|
429
|
-
|
|
474
|
+
`FileIndexCache` writes a temporary file and renames it over the cache file. A
|
|
475
|
+
missing file, invalid JSON, or data that fails context-fragment validation is
|
|
476
|
+
a cache miss, so the next `index()` call introspects again. `clock` is required
|
|
477
|
+
and controls file replacement retry delays; pass `globalThis` to use Node.js
|
|
478
|
+
timers. `dir` defaults to the OS temp directory. The optional `namespace`
|
|
479
|
+
separates cache generations; change it when the schema changes.
|
|
430
480
|
|
|
431
481
|
### Introspect once for concurrent callers
|
|
432
482
|
|
|
@@ -441,16 +491,18 @@ uses [`@zukhruf/mutex`](https://www.npmjs.com/package/@zukhruf/mutex):
|
|
|
441
491
|
import { Mutex } from '@zukhruf/mutex';
|
|
442
492
|
import { SqliteStore } from '@zukhruf/mutex/sqlite';
|
|
443
493
|
|
|
444
|
-
const mutex = new Mutex(
|
|
494
|
+
const mutex = new Mutex(
|
|
495
|
+
new SqliteStore('/var/lib/my-app/locks', { clock: globalThis }),
|
|
496
|
+
);
|
|
445
497
|
|
|
446
498
|
const fragments = await mutex.acquire('text2sql:main', () =>
|
|
447
499
|
text2sql.index({ names: ['main'] }),
|
|
448
500
|
);
|
|
449
501
|
```
|
|
450
502
|
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
503
|
+
Callers must share the cache directory, namespace, and adapter names. The first
|
|
504
|
+
caller introspects the database and writes the cache. When each waiting caller
|
|
505
|
+
acquires the same lock key, its `index()` call reads that cached schema.
|
|
454
506
|
|
|
455
507
|
`SqliteStore` shares keys between the processes on one host. For a fleet of
|
|
456
508
|
hosts, acquire the key from a service that all hosts share, for example a
|
|
@@ -460,7 +512,7 @@ Postgres advisory lock, and give all hosts the same cache.
|
|
|
460
512
|
> `FileIndexCache` parse-as-miss behavior tolerates a torn write: the next
|
|
461
513
|
> `index()` call introspects again.
|
|
462
514
|
|
|
463
|
-
## Streaming
|
|
515
|
+
## Streaming index progress
|
|
464
516
|
|
|
465
517
|
`AdapterIndexer#index({ onProgress })` emits progress events while it warms or reads
|
|
466
518
|
the schema cache for host-side indexing. The sandbox CLI writes the same event
|
|
@@ -510,7 +562,7 @@ imposing one duration policy.
|
|
|
510
562
|
`toSql()` is stateless. It reads the current schema fragments, but it does not
|
|
511
563
|
write messages, titles, or usage into your context store.
|
|
512
564
|
|
|
513
|
-
## Direct SQL
|
|
565
|
+
## Direct SQL generation with extra fragments
|
|
514
566
|
|
|
515
567
|
This lower-level helper still takes a single `adapter` because it bypasses the
|
|
516
568
|
multi-adapter `Text2Sql` wrapper entirely.
|
|
@@ -536,11 +588,11 @@ console.log(result.sql);
|
|
|
536
588
|
|
|
537
589
|
Full documentation available at [januarylabs.github.io/deepagents](https://januarylabs.github.io/deepagents/docs/text2sql):
|
|
538
590
|
|
|
539
|
-
- [Getting
|
|
591
|
+
- [Getting started](https://januarylabs.github.io/deepagents/docs/text2sql/getting-started)
|
|
540
592
|
- [Generate SQL](https://januarylabs.github.io/deepagents/docs/text2sql/to-sql)
|
|
541
|
-
- [Agent + SQLite
|
|
542
|
-
- [Teach the
|
|
543
|
-
- [Build
|
|
593
|
+
- [Agent + SQLite store recipe](https://januarylabs.github.io/deepagents/docs/text2sql/recipes/agent-sqlite-store)
|
|
594
|
+
- [Teach the system](https://januarylabs.github.io/deepagents/docs/text2sql/teach-the-system)
|
|
595
|
+
- [Build conversations](https://januarylabs.github.io/deepagents/docs/text2sql/build-conversations)
|
|
544
596
|
- [Grounding](https://januarylabs.github.io/deepagents/docs/text2sql/grounding)
|
|
545
597
|
- [PostgreSQL](https://januarylabs.github.io/deepagents/docs/text2sql/postgresql)
|
|
546
598
|
- [SQLite](https://januarylabs.github.io/deepagents/docs/text2sql/sqlite)
|
package/dist/bin/sql.js
CHANGED
|
@@ -10,16 +10,19 @@ import {
|
|
|
10
10
|
} from "@deepagents/context";
|
|
11
11
|
|
|
12
12
|
// packages/text2sql/src/lib/file-cache.ts
|
|
13
|
-
import {
|
|
13
|
+
import { atomicWrite } from "@zukhruf/fs";
|
|
14
|
+
import { createHash } from "node:crypto";
|
|
14
15
|
import { existsSync } from "node:fs";
|
|
15
|
-
import { mkdir, readFile
|
|
16
|
+
import { mkdir, readFile } from "node:fs/promises";
|
|
16
17
|
import { tmpdir } from "node:os";
|
|
17
18
|
import path from "node:path";
|
|
18
19
|
var FileCache = class {
|
|
19
20
|
path;
|
|
20
|
-
|
|
21
|
+
#clock;
|
|
22
|
+
constructor(watermark, { clock, extension = ".txt", baseDir = tmpdir() }) {
|
|
21
23
|
const hash = createHash("md5").update(watermark).digest("hex");
|
|
22
24
|
this.path = path.join(baseDir, `text2sql-${hash}${extension}`);
|
|
25
|
+
this.#clock = clock;
|
|
23
26
|
}
|
|
24
27
|
async get() {
|
|
25
28
|
if (existsSync(this.path)) {
|
|
@@ -29,9 +32,7 @@ var FileCache = class {
|
|
|
29
32
|
}
|
|
30
33
|
async set(content) {
|
|
31
34
|
await mkdir(path.dirname(this.path), { recursive: true });
|
|
32
|
-
|
|
33
|
-
await writeFile(tempPath, content, "utf-8");
|
|
34
|
-
await rename(tempPath, this.path);
|
|
35
|
+
await atomicWrite(this.path, content, { clock: this.#clock });
|
|
35
36
|
}
|
|
36
37
|
};
|
|
37
38
|
var JsonCache = class extends FileCache {
|
|
@@ -40,8 +41,8 @@ var JsonCache = class extends FileCache {
|
|
|
40
41
|
* @param isValid - Checks that a parsed cache file holds a T; a file that
|
|
41
42
|
* fails it is read as a miss.
|
|
42
43
|
*/
|
|
43
|
-
constructor(watermark, isValid, baseDir) {
|
|
44
|
-
super(watermark, ".json", baseDir);
|
|
44
|
+
constructor(watermark, isValid, { clock, baseDir }) {
|
|
45
|
+
super(watermark, { clock, extension: ".json", baseDir });
|
|
45
46
|
this.#isValid = isValid;
|
|
46
47
|
}
|
|
47
48
|
async read() {
|
|
@@ -63,9 +64,11 @@ var JsonCache = class extends FileCache {
|
|
|
63
64
|
|
|
64
65
|
// packages/text2sql/src/lib/index-cache.ts
|
|
65
66
|
var FileIndexCache = class {
|
|
67
|
+
#clock;
|
|
66
68
|
#dir;
|
|
67
69
|
#namespace;
|
|
68
|
-
constructor(options
|
|
70
|
+
constructor(options) {
|
|
71
|
+
this.#clock = options.clock;
|
|
69
72
|
this.#dir = options.dir;
|
|
70
73
|
this.#namespace = options.namespace;
|
|
71
74
|
}
|
|
@@ -77,7 +80,10 @@ var FileIndexCache = class {
|
|
|
77
80
|
}
|
|
78
81
|
#cacheFor(key) {
|
|
79
82
|
const watermark = this.#namespace ? `${this.#namespace}-${key}` : key;
|
|
80
|
-
return new JsonCache(watermark, isFragmentList,
|
|
83
|
+
return new JsonCache(watermark, isFragmentList, {
|
|
84
|
+
clock: this.#clock,
|
|
85
|
+
baseDir: this.#dir
|
|
86
|
+
});
|
|
81
87
|
}
|
|
82
88
|
};
|
|
83
89
|
function isFragmentList(value) {
|
|
@@ -108,6 +114,7 @@ function validateAdapterNames(names) {
|
|
|
108
114
|
}
|
|
109
115
|
|
|
110
116
|
// packages/text2sql/src/lib/adapters/adapter.ts
|
|
117
|
+
import pMemoize from "p-memoize";
|
|
111
118
|
import { format as formatSql } from "sql-formatter";
|
|
112
119
|
import { z } from "zod";
|
|
113
120
|
|
|
@@ -555,7 +562,7 @@ import {
|
|
|
555
562
|
mkdir as mkdir2,
|
|
556
563
|
open,
|
|
557
564
|
stat,
|
|
558
|
-
writeFile
|
|
565
|
+
writeFile
|
|
559
566
|
} from "node:fs/promises";
|
|
560
567
|
import * as path3 from "node:path";
|
|
561
568
|
import { finished } from "node:stream/promises";
|
|
@@ -621,7 +628,7 @@ var IndexCommand = class extends SqlCommand {
|
|
|
621
628
|
names,
|
|
622
629
|
onProgress: createProgressHandler(eventsStream, verbose, ctx.stderr)
|
|
623
630
|
});
|
|
624
|
-
await
|
|
631
|
+
await writeFile(fragmentsPath, JSON.stringify(fragments, null, 2));
|
|
625
632
|
const manifest = {
|
|
626
633
|
fragmentsPath,
|
|
627
634
|
eventsPath,
|
|
@@ -734,7 +741,7 @@ function dedupe(values) {
|
|
|
734
741
|
}
|
|
735
742
|
|
|
736
743
|
// packages/text2sql/src/bin/commands/run.ts
|
|
737
|
-
import { mkdir as mkdir3, writeFile as
|
|
744
|
+
import { mkdir as mkdir3, writeFile as writeFile2 } from "node:fs/promises";
|
|
738
745
|
import * as path4 from "node:path";
|
|
739
746
|
import { v7 as v72 } from "uuid";
|
|
740
747
|
var RunCommand = class extends SqlQueryCommand {
|
|
@@ -749,7 +756,7 @@ var RunCommand = class extends SqlQueryCommand {
|
|
|
749
756
|
const outPath = path4.join(outputDir, `${v72()}.json`);
|
|
750
757
|
try {
|
|
751
758
|
await mkdir3(outputDir, { recursive: true });
|
|
752
|
-
await
|
|
759
|
+
await writeFile2(outPath, JSON.stringify(rows, null, 2));
|
|
753
760
|
} catch (error) {
|
|
754
761
|
this.fail(errorMessage2(error));
|
|
755
762
|
}
|
|
@@ -888,7 +895,7 @@ function resolveIndexCache() {
|
|
|
888
895
|
if (!dir && !namespace) {
|
|
889
896
|
return void 0;
|
|
890
897
|
}
|
|
891
|
-
return new FileIndexCache({ dir, namespace });
|
|
898
|
+
return new FileIndexCache({ clock: globalThis, dir, namespace });
|
|
892
899
|
}
|
|
893
900
|
async function runCommand(command, positional, options) {
|
|
894
901
|
let ctx;
|
|
@@ -1,18 +1,2 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Convert HuggingFace model to GGUF format using Docker
|
|
3
|
-
* Uses the official llama.cpp image - no local installation needed
|
|
4
|
-
*
|
|
5
|
-
* Usage: node convert-to-gguf.ts <model_dir> <output.gguf> [quantization]
|
|
6
|
-
*
|
|
7
|
-
* Arguments:
|
|
8
|
-
* model_dir Path to HuggingFace model directory (contains model.safetensors)
|
|
9
|
-
* output.gguf Output path for the GGUF file
|
|
10
|
-
* quantization Quantization type (default: q8_0)
|
|
11
|
-
* Options: f32, f16, bf16, q8_0, q4_0, q4_1, q5_0, q5_1
|
|
12
|
-
*
|
|
13
|
-
* Examples:
|
|
14
|
-
* node convert-to-gguf.ts ./qwen3-sql-final ./qwen3-sql.gguf
|
|
15
|
-
* node convert-to-gguf.ts ./qwen3-sql-final ./qwen3-sql-q4.gguf q4_0
|
|
16
|
-
*/
|
|
17
1
|
export {};
|
|
18
2
|
//# sourceMappingURL=convert-to-gguf.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"convert-to-gguf.d.ts","sourceRoot":"","sources":["../../src/finetune/convert-to-gguf.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"convert-to-gguf.d.ts","sourceRoot":"","sources":["../../src/finetune/convert-to-gguf.ts"],"names":[],"mappings":""}
|
package/dist/index.js
CHANGED
|
@@ -19,6 +19,7 @@ function validateAdapterNames(names) {
|
|
|
19
19
|
}
|
|
20
20
|
|
|
21
21
|
// packages/text2sql/src/lib/adapters/adapter.ts
|
|
22
|
+
import pMemoize from "p-memoize";
|
|
22
23
|
import { format as formatSql } from "sql-formatter";
|
|
23
24
|
import { z } from "zod";
|
|
24
25
|
|
|
@@ -431,12 +432,7 @@ var Adapter = class {
|
|
|
431
432
|
#decodeShellEscapes(sql) {
|
|
432
433
|
return sql.replace(/\\n/g, "\n").replace(/\\t/g, " ").replace(/\\([().*$])/g, "$1").replace(/\\(?=$)/gm, "");
|
|
433
434
|
}
|
|
434
|
-
#
|
|
435
|
-
async #resolveScope() {
|
|
436
|
-
if (this.#cachedAllowedEntities) return this.#cachedAllowedEntities;
|
|
437
|
-
this.#cachedAllowedEntities = await this.resolveAllowedEntities();
|
|
438
|
-
return this.#cachedAllowedEntities;
|
|
439
|
-
}
|
|
435
|
+
#resolveScope = pMemoize(() => this.resolveAllowedEntities());
|
|
440
436
|
async #analyzePolicy(sql) {
|
|
441
437
|
return this.#policyAnalyzer.analyze(sql, {
|
|
442
438
|
resolveAllowedEntities: () => this.#resolveScope()
|
|
@@ -1129,16 +1125,19 @@ import {
|
|
|
1129
1125
|
} from "@deepagents/context";
|
|
1130
1126
|
|
|
1131
1127
|
// packages/text2sql/src/lib/file-cache.ts
|
|
1132
|
-
import {
|
|
1128
|
+
import { atomicWrite } from "@zukhruf/fs";
|
|
1129
|
+
import { createHash } from "node:crypto";
|
|
1133
1130
|
import { existsSync } from "node:fs";
|
|
1134
|
-
import { mkdir, readFile
|
|
1131
|
+
import { mkdir, readFile } from "node:fs/promises";
|
|
1135
1132
|
import { tmpdir } from "node:os";
|
|
1136
1133
|
import path from "node:path";
|
|
1137
1134
|
var FileCache = class {
|
|
1138
1135
|
path;
|
|
1139
|
-
|
|
1136
|
+
#clock;
|
|
1137
|
+
constructor(watermark, { clock, extension = ".txt", baseDir = tmpdir() }) {
|
|
1140
1138
|
const hash2 = createHash("md5").update(watermark).digest("hex");
|
|
1141
1139
|
this.path = path.join(baseDir, `text2sql-${hash2}${extension}`);
|
|
1140
|
+
this.#clock = clock;
|
|
1142
1141
|
}
|
|
1143
1142
|
async get() {
|
|
1144
1143
|
if (existsSync(this.path)) {
|
|
@@ -1148,9 +1147,7 @@ var FileCache = class {
|
|
|
1148
1147
|
}
|
|
1149
1148
|
async set(content) {
|
|
1150
1149
|
await mkdir(path.dirname(this.path), { recursive: true });
|
|
1151
|
-
|
|
1152
|
-
await writeFile(tempPath, content, "utf-8");
|
|
1153
|
-
await rename(tempPath, this.path);
|
|
1150
|
+
await atomicWrite(this.path, content, { clock: this.#clock });
|
|
1154
1151
|
}
|
|
1155
1152
|
};
|
|
1156
1153
|
var JsonCache = class extends FileCache {
|
|
@@ -1159,8 +1156,8 @@ var JsonCache = class extends FileCache {
|
|
|
1159
1156
|
* @param isValid - Checks that a parsed cache file holds a T; a file that
|
|
1160
1157
|
* fails it is read as a miss.
|
|
1161
1158
|
*/
|
|
1162
|
-
constructor(watermark, isValid, baseDir) {
|
|
1163
|
-
super(watermark, ".json", baseDir);
|
|
1159
|
+
constructor(watermark, isValid, { clock, baseDir }) {
|
|
1160
|
+
super(watermark, { clock, extension: ".json", baseDir });
|
|
1164
1161
|
this.#isValid = isValid;
|
|
1165
1162
|
}
|
|
1166
1163
|
async read() {
|
|
@@ -1182,9 +1179,11 @@ var JsonCache = class extends FileCache {
|
|
|
1182
1179
|
|
|
1183
1180
|
// packages/text2sql/src/lib/index-cache.ts
|
|
1184
1181
|
var FileIndexCache = class {
|
|
1182
|
+
#clock;
|
|
1185
1183
|
#dir;
|
|
1186
1184
|
#namespace;
|
|
1187
|
-
constructor(options
|
|
1185
|
+
constructor(options) {
|
|
1186
|
+
this.#clock = options.clock;
|
|
1188
1187
|
this.#dir = options.dir;
|
|
1189
1188
|
this.#namespace = options.namespace;
|
|
1190
1189
|
}
|
|
@@ -1196,7 +1195,10 @@ var FileIndexCache = class {
|
|
|
1196
1195
|
}
|
|
1197
1196
|
#cacheFor(key) {
|
|
1198
1197
|
const watermark = this.#namespace ? `${this.#namespace}-${key}` : key;
|
|
1199
|
-
return new JsonCache(watermark, isFragmentList,
|
|
1198
|
+
return new JsonCache(watermark, isFragmentList, {
|
|
1199
|
+
clock: this.#clock,
|
|
1200
|
+
baseDir: this.#dir
|
|
1201
|
+
});
|
|
1200
1202
|
}
|
|
1201
1203
|
};
|
|
1202
1204
|
function isFragmentList(value) {
|
|
@@ -1367,8 +1369,9 @@ function errorMessage(error) {
|
|
|
1367
1369
|
}
|
|
1368
1370
|
|
|
1369
1371
|
// packages/text2sql/src/lib/checkpoint.ts
|
|
1372
|
+
import { durableWrite } from "@zukhruf/fs";
|
|
1370
1373
|
import { createHash as createHash2 } from "node:crypto";
|
|
1371
|
-
import { existsSync as existsSync2, readFileSync
|
|
1374
|
+
import { existsSync as existsSync2, readFileSync } from "node:fs";
|
|
1372
1375
|
import pLimit from "p-limit";
|
|
1373
1376
|
import z4 from "zod";
|
|
1374
1377
|
function jsonCodec(schema) {
|
|
@@ -1388,24 +1391,29 @@ var Checkpoint = class _Checkpoint {
|
|
|
1388
1391
|
points;
|
|
1389
1392
|
path;
|
|
1390
1393
|
configHash;
|
|
1391
|
-
|
|
1394
|
+
#clock;
|
|
1395
|
+
#writes = pLimit(1);
|
|
1396
|
+
#changes = 0;
|
|
1397
|
+
#writtenChanges = 0;
|
|
1398
|
+
constructor(path6, configHash, clock, points) {
|
|
1392
1399
|
this.points = points;
|
|
1393
1400
|
this.path = path6;
|
|
1394
1401
|
this.configHash = configHash;
|
|
1402
|
+
this.#clock = clock;
|
|
1395
1403
|
}
|
|
1396
1404
|
/**
|
|
1397
1405
|
* Load checkpoint from file, or return empty checkpoint if none exists.
|
|
1398
1406
|
* Handles corrupted files and config changes gracefully.
|
|
1399
1407
|
*/
|
|
1400
1408
|
static async load(options) {
|
|
1401
|
-
const { path: path6, configHash } = options;
|
|
1409
|
+
const { path: path6, configHash, clock } = options;
|
|
1402
1410
|
if (existsSync2(path6)) {
|
|
1403
1411
|
try {
|
|
1404
1412
|
const content = readFileSync(path6, "utf-8");
|
|
1405
1413
|
const file = checkpointFile.parse(JSON.parse(content));
|
|
1406
1414
|
if (configHash && file.configHash && file.configHash !== configHash) {
|
|
1407
1415
|
console.log("\u26A0 Config changed, starting fresh");
|
|
1408
|
-
return new _Checkpoint(path6, configHash, {});
|
|
1416
|
+
return new _Checkpoint(path6, configHash, clock, {});
|
|
1409
1417
|
}
|
|
1410
1418
|
const points = file.points;
|
|
1411
1419
|
const totalEntries = Object.values(points).reduce(
|
|
@@ -1413,14 +1421,14 @@ var Checkpoint = class _Checkpoint {
|
|
|
1413
1421
|
0
|
|
1414
1422
|
);
|
|
1415
1423
|
console.log(`\u2713 Resuming from checkpoint (${totalEntries} entries)`);
|
|
1416
|
-
return new _Checkpoint(path6, configHash, points);
|
|
1424
|
+
return new _Checkpoint(path6, configHash, clock, points);
|
|
1417
1425
|
} catch {
|
|
1418
1426
|
console.log("\u26A0 Checkpoint corrupted, starting fresh");
|
|
1419
|
-
return new _Checkpoint(path6, configHash, {});
|
|
1427
|
+
return new _Checkpoint(path6, configHash, clock, {});
|
|
1420
1428
|
}
|
|
1421
1429
|
}
|
|
1422
1430
|
console.log("Starting new checkpoint");
|
|
1423
|
-
return new _Checkpoint(path6, configHash, {});
|
|
1431
|
+
return new _Checkpoint(path6, configHash, clock, {});
|
|
1424
1432
|
}
|
|
1425
1433
|
/**
|
|
1426
1434
|
* Run a single computation with checkpointing.
|
|
@@ -1484,15 +1492,25 @@ var Checkpoint = class _Checkpoint {
|
|
|
1484
1492
|
getPath() {
|
|
1485
1493
|
return this.path;
|
|
1486
1494
|
}
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1495
|
+
/**
|
|
1496
|
+
* Writes one snapshot at a time, so an older snapshot never lands after a
|
|
1497
|
+
* newer one. Each write takes the snapshot when it starts, so a save that
|
|
1498
|
+
* waited behind a write that already holds its change writes nothing.
|
|
1499
|
+
*/
|
|
1500
|
+
save() {
|
|
1501
|
+
this.#changes += 1;
|
|
1502
|
+
return this.#writes(async () => {
|
|
1503
|
+
const changes = this.#changes;
|
|
1504
|
+
if (changes === this.#writtenChanges) return;
|
|
1505
|
+
const file = {
|
|
1506
|
+
configHash: this.configHash,
|
|
1507
|
+
points: this.points
|
|
1508
|
+
};
|
|
1509
|
+
await durableWrite(this.path, JSON.stringify(file, null, 2), {
|
|
1510
|
+
clock: this.#clock
|
|
1511
|
+
});
|
|
1512
|
+
this.#writtenChanges = changes;
|
|
1513
|
+
});
|
|
1496
1514
|
}
|
|
1497
1515
|
};
|
|
1498
1516
|
function hash(value) {
|
|
@@ -1546,9 +1564,9 @@ function hashConfig(config) {
|
|
|
1546
1564
|
|
|
1547
1565
|
// packages/text2sql/src/lib/fs/sqlite/sqlite-fs.ts
|
|
1548
1566
|
import { Declaration } from "@zukhruf/sqlite-drift";
|
|
1567
|
+
import { open } from "@zukhruf/sqlite-open";
|
|
1549
1568
|
import { transaction } from "@zukhruf/sqlite-transaction";
|
|
1550
1569
|
import * as path2 from "node:path";
|
|
1551
|
-
import { DatabaseSync } from "node:sqlite";
|
|
1552
1570
|
import z6 from "zod";
|
|
1553
1571
|
|
|
1554
1572
|
// packages/text2sql/src/lib/fs/columns.ts
|
|
@@ -1558,7 +1576,6 @@ var bigintColumn = z5.union([z5.number(), z5.string().regex(/^-?\d+$/)]).transfo
|
|
|
1558
1576
|
|
|
1559
1577
|
// packages/text2sql/src/lib/fs/sqlite/ddl.sqlite-fs.ts
|
|
1560
1578
|
var SETUP = `
|
|
1561
|
-
PRAGMA journal_mode = WAL;
|
|
1562
1579
|
PRAGMA synchronous = NORMAL;
|
|
1563
1580
|
PRAGMA cache_size = -64000;
|
|
1564
1581
|
PRAGMA temp_store = MEMORY;
|
|
@@ -1620,7 +1637,7 @@ var SqliteFs = class {
|
|
|
1620
1637
|
this.#chunkSize = options.chunkSize ?? 1024 * 1024;
|
|
1621
1638
|
const normalizedRoot = this.#normalizeRoot(options.root);
|
|
1622
1639
|
this.#root = normalizedRoot === "/" ? "" : normalizedRoot;
|
|
1623
|
-
this.#db =
|
|
1640
|
+
this.#db = open(options.dbPath);
|
|
1624
1641
|
this.#db.exec(SETUP);
|
|
1625
1642
|
tables.apply(this.#db);
|
|
1626
1643
|
this.#sql = this.#db.createTagStore();
|