@deepagents/text2sql 7.2.0 → 7.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/README.md +80 -28
  2. package/dist/bin/sql.js +22 -15
  3. package/dist/finetune/convert-to-gguf.d.ts +0 -16
  4. package/dist/finetune/convert-to-gguf.d.ts.map +1 -1
  5. package/dist/index.js +52 -35
  6. package/dist/index.js.map +2 -2
  7. package/dist/lib/adapters/adapter.d.ts.map +1 -1
  8. package/dist/lib/adapters/bigquery/index.js +2 -6
  9. package/dist/lib/adapters/bigquery/index.js.map +2 -2
  10. package/dist/lib/adapters/clickhouse/index.js +4 -8
  11. package/dist/lib/adapters/clickhouse/index.js.map +3 -3
  12. package/dist/lib/adapters/duckdb/index.js +6 -10
  13. package/dist/lib/adapters/duckdb/index.js.map +3 -3
  14. package/dist/lib/adapters/mysql/index.js +2 -6
  15. package/dist/lib/adapters/mysql/index.js.map +2 -2
  16. package/dist/lib/adapters/postgres/index.js +4 -8
  17. package/dist/lib/adapters/postgres/index.js.map +3 -3
  18. package/dist/lib/adapters/posthog/index.js +2 -6
  19. package/dist/lib/adapters/posthog/index.js.map +2 -2
  20. package/dist/lib/adapters/spreadsheet/index.js +7 -13
  21. package/dist/lib/adapters/spreadsheet/index.js.map +3 -3
  22. package/dist/lib/adapters/spreadsheet/spreadsheet.d.ts.map +1 -1
  23. package/dist/lib/adapters/sqlite/index.js +5 -11
  24. package/dist/lib/adapters/sqlite/index.js.map +3 -3
  25. package/dist/lib/adapters/sqlite/table.sqlite.grounding.d.ts.map +1 -1
  26. package/dist/lib/adapters/sqlserver/index.js +2 -6
  27. package/dist/lib/adapters/sqlserver/index.js.map +2 -2
  28. package/dist/lib/checkpoint.d.ts +9 -0
  29. package/dist/lib/checkpoint.d.ts.map +1 -1
  30. package/dist/lib/file-cache.d.ts +12 -2
  31. package/dist/lib/file-cache.d.ts.map +1 -1
  32. package/dist/lib/fs/sqlite/ddl.sqlite-fs.d.ts +1 -1
  33. package/dist/lib/fs/sqlite/ddl.sqlite-fs.d.ts.map +1 -1
  34. package/dist/lib/fs/sqlite/sqlite-fs.d.ts.map +1 -1
  35. package/dist/lib/index-cache.d.ts +4 -1
  36. package/dist/lib/index-cache.d.ts.map +1 -1
  37. package/package.json +15 -9
package/README.md CHANGED
@@ -4,12 +4,12 @@ AI-powered natural language to SQL. Ask questions in plain English, get executab
4
4
 
5
5
  ## Features
6
6
 
7
- - **Natural Language to SQL** - Convert questions to validated, executable queries
8
- - **Multi-Database Support** - PostgreSQL, SQLite, DuckDB, SQL Server, MySQL/MariaDB, BigQuery, ClickHouse, and PostHog HogQL adapters
9
- - **Schema-Aware** - Automatic introspection of tables, relationships, indexes, and constraints
10
- - **Domain Knowledge** - Inject business terms, guardrails, and query patterns via fragments
7
+ - **Natural language to SQL** - Convert questions to validated, executable queries
8
+ - **Database adapters** - PostgreSQL, SQLite, DuckDB, SQL Server, MySQL/MariaDB, BigQuery, ClickHouse, and PostHog HogQL adapters
9
+ - **Schema introspection** - Automatic introspection of tables, relationships, indexes, and constraints
10
+ - **Domain knowledge** - Inject business terms, guardrails, and query patterns via fragments
11
11
  - **Conversational** - Multi-turn conversations with context persistence
12
- - **Safe by Default** - Read-only queries, validation, and configurable guardrails
12
+ - **Safe by default** - Read-only queries, validation, and configurable guardrails
13
13
 
14
14
  ## Installation
15
15
 
@@ -29,9 +29,24 @@ npm install @clickhouse/client # ClickHouse, or use your preferred client
29
29
  # PostHog uses native fetch and needs no additional client package
30
30
  ```
31
31
 
32
- Requires Node.js LTS
32
+ The spreadsheet adapter requires `xlsx` for Excel, CSV, and TSV files. It is
33
+ an optional peer dependency; install it only if you use
34
+ `@deepagents/text2sql/spreadsheet`. With npm 12, add this to your project's
35
+ `.npmrc` to allow directly declared URL dependencies:
33
36
 
34
- ## Quick Start
37
+ ```ini
38
+ allow-remote=root
39
+ ```
40
+
41
+ Then install SheetJS `xlsx` 0.20.3 from its CDN:
42
+
43
+ ```bash
44
+ npm install https://cdn.sheetjs.com/xlsx-0.20.3/xlsx-0.20.3.tgz
45
+ ```
46
+
47
+ Requires Node.js LTS.
48
+
49
+ ## Quick start
35
50
 
36
51
  ```typescript
37
52
  import { groq } from '@ai-sdk/groq';
@@ -96,11 +111,11 @@ maps dynamically, use `isValidAdapterName(name)` to check one key or
96
111
  `validateAdapterNames(names)` to fail fast before constructing `Text2Sql` or
97
112
  the sandbox-side adapter module used by the `sql` CLI.
98
113
 
99
- ## AI Model Providers
114
+ ## AI model providers
100
115
 
101
116
  Text2SQL works with any model provider supported by the [Vercel AI SDK](https://sdk.vercel.ai/docs), including OpenAI, Anthropic, Google, Groq, and more.
102
117
 
103
- ## Building a Conversational Agent
118
+ ## Building a conversational agent
104
119
 
105
120
  `Text2Sql` owns SQL generation and schema indexing, but you own the chat loop.
106
121
  Build the agent by composing a `ContextEngine`, a sandbox, and `agent` +
@@ -144,6 +159,7 @@ const backend = await createDockerSandbox({
144
159
  ],
145
160
  env: {
146
161
  TEXT2SQL_ADAPTERS: '/workspace/text2sql-adapters.ts',
162
+ TEXT2SQL_OUT_DIR: '/tmp/text2sql',
147
163
  },
148
164
  });
149
165
  const sandbox = await createBashTool({
@@ -179,7 +195,12 @@ for await (const chunk of stream) {
179
195
 
180
196
  The `/workspace/text2sql-adapters.ts` module must exist in the sandbox and
181
197
  default-export your adapter map. Mount your project into `/workspace` (as above)
182
- or upload/write that module before you call `sql index` or `chat()`.
198
+ or upload or write that module before calling `sql index`. With a remote Docker
199
+ daemon, the bind mount's `hostPath` must exist on the daemon's host.
200
+
201
+ `TEXT2SQL_OUT_DIR` keeps schema and query artifacts in the sandbox's writable
202
+ `/tmp/text2sql` directory. The default `./sql` would be inside the read-only
203
+ `/workspace` mount.
183
204
 
184
205
  `instructions()` returns the SQL-flavored system fragments (policies, workflows,
185
206
  clarifications, style guides) — spread them into `context.set()` alongside the
@@ -197,7 +218,7 @@ turns, the reminder is persisted as a synthetic user message for replay and
197
218
  prompt caching; bare `createPrepareStep()` integrations and
198
219
  `agent.generate()` own persistence of their generated assistant history.
199
220
 
200
- ## Advanced: SQL CLI in Sandboxes
221
+ ## Advanced: SQL CLI in sandboxes
201
222
 
202
223
  `sql validate <db> "..."` and `sql run <db> "..."` are real commands from the
203
224
  `@deepagents/text2sql` package. `<db>` is the configured adapter name. With a
@@ -228,11 +249,13 @@ changes). With neither set, every `sql index` introspects fresh. See
228
249
  [Schema index caching](#schema-index-caching) for
229
250
  the underlying injectable primitives.
230
251
 
231
- For in-process or virtual-sandbox usage (without installing the package CLI),
232
- wrap an existing `Text2Sql` instance as a just-bash custom command:
252
+ For in-process or virtual-sandbox usage, install `just-bash` and wrap an existing
253
+ `Text2Sql` instance as a custom command:
233
254
 
234
255
  ```typescript
235
- import { InMemoryFs, createVirtualSandbox } from '@deepagents/context';
256
+ import { InMemoryFs } from 'just-bash';
257
+
258
+ import { createVirtualSandbox } from '@deepagents/context';
236
259
  import {
237
260
  type CreateSqlCommandOptions,
238
261
  type CreateSqlCommandResult,
@@ -396,6 +419,26 @@ for await (const chunk of followUp) {
396
419
  }
397
420
  ```
398
421
 
422
+ ## Checkpoint files
423
+
424
+ `Checkpoint` from `@deepagents/text2sql` saves completed computations so a
425
+ pipeline can resume without repeating them. Load a file with
426
+ `Checkpoint.load({ path, configHash, clock: globalThis })`, then use `run()` for
427
+ one computation or `each()` for a sequence of inputs. Both require a codec to
428
+ encode outputs and validate them on resume; `jsonCodec(schema)` handles plain
429
+ JSON values. The optional `configHash` invalidates saved results when both the
430
+ file and the caller supply nonempty hashes that differ.
431
+
432
+ `clock` is required and controls the delay between file replacement retries
433
+ when a file is busy. Pass `globalThis` to use Node.js timers.
434
+
435
+ Writes from one `Checkpoint` instance run in order. Each write captures the
436
+ latest state when it starts, so an earlier snapshot cannot overwrite a later
437
+ one. The checkpoint uses `durableWrite` to flush the file before replacing it and
438
+ sync the parent directory where supported. Await `run()`, `each()`, or
439
+ `point().through()` before exiting. Separate instances or processes do not
440
+ coordinate writes to the same file.
441
+
399
442
  ## Schema index caching
400
443
 
401
444
  Schema introspection is the expensive part of indexing. `Text2Sql` and
@@ -420,13 +463,20 @@ import { FileIndexCache, Text2Sql } from '@deepagents/text2sql';
420
463
 
421
464
  const text2sql = new Text2Sql({
422
465
  adapters: { main: adapter },
423
- cache: new FileIndexCache({ dir: '/var/cache/text2sql', namespace: 'v1' }),
466
+ cache: new FileIndexCache({
467
+ clock: globalThis,
468
+ dir: '/var/cache/text2sql',
469
+ namespace: 'v1',
470
+ }),
424
471
  });
425
472
  ```
426
473
 
427
- `FileIndexCache` writes atomically (temp + rename). It treats a file that does
428
- not parse to context fragments as a miss, so a torn read causes a new
429
- introspection. `dir` defaults to the OS temp directory.
474
+ `FileIndexCache` writes a temporary file and renames it over the cache file. A
475
+ missing file, invalid JSON, or data that fails context-fragment validation is
476
+ a cache miss, so the next `index()` call introspects again. `clock` is required
477
+ and controls file replacement retry delays; pass `globalThis` to use Node.js
478
+ timers. `dir` defaults to the OS temp directory. The optional `namespace`
479
+ separates cache generations; change it when the schema changes.
430
480
 
431
481
  ### Introspect once for concurrent callers
432
482
 
@@ -441,16 +491,18 @@ uses [`@zukhruf/mutex`](https://www.npmjs.com/package/@zukhruf/mutex):
441
491
  import { Mutex } from '@zukhruf/mutex';
442
492
  import { SqliteStore } from '@zukhruf/mutex/sqlite';
443
493
 
444
- const mutex = new Mutex(new SqliteStore('/var/lib/my-app/locks'));
494
+ const mutex = new Mutex(
495
+ new SqliteStore('/var/lib/my-app/locks', { clock: globalThis }),
496
+ );
445
497
 
446
498
  const fragments = await mutex.acquire('text2sql:main', () =>
447
499
  text2sql.index({ names: ['main'] }),
448
500
  );
449
501
  ```
450
502
 
451
- The first caller introspects the database and writes the cache. The other
452
- callers wait. When each waiter gets the key, its `index()` call reads the cache
453
- that the first caller wrote.
503
+ Callers must share the cache directory, namespace, and adapter names. The first
504
+ caller introspects the database and writes the cache. When each waiting caller
505
+ acquires the same lock key, its `index()` call reads that cached schema.
454
506
 
455
507
  `SqliteStore` shares keys between the processes on one host. For a fleet of
456
508
  hosts, acquire the key from a service that all hosts share, for example a
@@ -460,7 +512,7 @@ Postgres advisory lock, and give all hosts the same cache.
460
512
  > `FileIndexCache` parse-as-miss behavior tolerates a torn write: the next
461
513
  > `index()` call introspects again.
462
514
 
463
- ## Streaming Index Progress
515
+ ## Streaming index progress
464
516
 
465
517
  `AdapterIndexer#index({ onProgress })` emits progress events while it warms or reads
466
518
  the schema cache for host-side indexing. The sandbox CLI writes the same event
@@ -510,7 +562,7 @@ imposing one duration policy.
510
562
  `toSql()` is stateless. It reads the current schema fragments, but it does not
511
563
  write messages, titles, or usage into your context store.
512
564
 
513
- ## Direct SQL Generation with Extra Fragments
565
+ ## Direct SQL generation with extra fragments
514
566
 
515
567
  This lower-level helper still takes a single `adapter` because it bypasses the
516
568
  multi-adapter `Text2Sql` wrapper entirely.
@@ -536,11 +588,11 @@ console.log(result.sql);
536
588
 
537
589
  Full documentation available at [januarylabs.github.io/deepagents](https://januarylabs.github.io/deepagents/docs/text2sql):
538
590
 
539
- - [Getting Started](https://januarylabs.github.io/deepagents/docs/text2sql/getting-started)
591
+ - [Getting started](https://januarylabs.github.io/deepagents/docs/text2sql/getting-started)
540
592
  - [Generate SQL](https://januarylabs.github.io/deepagents/docs/text2sql/to-sql)
541
- - [Agent + SQLite Store Recipe](https://januarylabs.github.io/deepagents/docs/text2sql/recipes/agent-sqlite-store)
542
- - [Teach the System](https://januarylabs.github.io/deepagents/docs/text2sql/teach-the-system)
543
- - [Build Conversations](https://januarylabs.github.io/deepagents/docs/text2sql/build-conversations)
593
+ - [Agent + SQLite store recipe](https://januarylabs.github.io/deepagents/docs/text2sql/recipes/agent-sqlite-store)
594
+ - [Teach the system](https://januarylabs.github.io/deepagents/docs/text2sql/teach-the-system)
595
+ - [Build conversations](https://januarylabs.github.io/deepagents/docs/text2sql/build-conversations)
544
596
  - [Grounding](https://januarylabs.github.io/deepagents/docs/text2sql/grounding)
545
597
  - [PostgreSQL](https://januarylabs.github.io/deepagents/docs/text2sql/postgresql)
546
598
  - [SQLite](https://januarylabs.github.io/deepagents/docs/text2sql/sqlite)
package/dist/bin/sql.js CHANGED
@@ -10,16 +10,19 @@ import {
10
10
  } from "@deepagents/context";
11
11
 
12
12
  // packages/text2sql/src/lib/file-cache.ts
13
- import { createHash, randomUUID } from "node:crypto";
13
+ import { atomicWrite } from "@zukhruf/fs";
14
+ import { createHash } from "node:crypto";
14
15
  import { existsSync } from "node:fs";
15
- import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
16
+ import { mkdir, readFile } from "node:fs/promises";
16
17
  import { tmpdir } from "node:os";
17
18
  import path from "node:path";
18
19
  var FileCache = class {
19
20
  path;
20
- constructor(watermark, extension = ".txt", baseDir = tmpdir()) {
21
+ #clock;
22
+ constructor(watermark, { clock, extension = ".txt", baseDir = tmpdir() }) {
21
23
  const hash = createHash("md5").update(watermark).digest("hex");
22
24
  this.path = path.join(baseDir, `text2sql-${hash}${extension}`);
25
+ this.#clock = clock;
23
26
  }
24
27
  async get() {
25
28
  if (existsSync(this.path)) {
@@ -29,9 +32,7 @@ var FileCache = class {
29
32
  }
30
33
  async set(content) {
31
34
  await mkdir(path.dirname(this.path), { recursive: true });
32
- const tempPath = `${this.path}.${randomUUID()}.tmp`;
33
- await writeFile(tempPath, content, "utf-8");
34
- await rename(tempPath, this.path);
35
+ await atomicWrite(this.path, content, { clock: this.#clock });
35
36
  }
36
37
  };
37
38
  var JsonCache = class extends FileCache {
@@ -40,8 +41,8 @@ var JsonCache = class extends FileCache {
40
41
  * @param isValid - Checks that a parsed cache file holds a T; a file that
41
42
  * fails it is read as a miss.
42
43
  */
43
- constructor(watermark, isValid, baseDir) {
44
- super(watermark, ".json", baseDir);
44
+ constructor(watermark, isValid, { clock, baseDir }) {
45
+ super(watermark, { clock, extension: ".json", baseDir });
45
46
  this.#isValid = isValid;
46
47
  }
47
48
  async read() {
@@ -63,9 +64,11 @@ var JsonCache = class extends FileCache {
63
64
 
64
65
  // packages/text2sql/src/lib/index-cache.ts
65
66
  var FileIndexCache = class {
67
+ #clock;
66
68
  #dir;
67
69
  #namespace;
68
- constructor(options = {}) {
70
+ constructor(options) {
71
+ this.#clock = options.clock;
69
72
  this.#dir = options.dir;
70
73
  this.#namespace = options.namespace;
71
74
  }
@@ -77,7 +80,10 @@ var FileIndexCache = class {
77
80
  }
78
81
  #cacheFor(key) {
79
82
  const watermark = this.#namespace ? `${this.#namespace}-${key}` : key;
80
- return new JsonCache(watermark, isFragmentList, this.#dir);
83
+ return new JsonCache(watermark, isFragmentList, {
84
+ clock: this.#clock,
85
+ baseDir: this.#dir
86
+ });
81
87
  }
82
88
  };
83
89
  function isFragmentList(value) {
@@ -108,6 +114,7 @@ function validateAdapterNames(names) {
108
114
  }
109
115
 
110
116
  // packages/text2sql/src/lib/adapters/adapter.ts
117
+ import pMemoize from "p-memoize";
111
118
  import { format as formatSql } from "sql-formatter";
112
119
  import { z } from "zod";
113
120
 
@@ -555,7 +562,7 @@ import {
555
562
  mkdir as mkdir2,
556
563
  open,
557
564
  stat,
558
- writeFile as writeFile2
565
+ writeFile
559
566
  } from "node:fs/promises";
560
567
  import * as path3 from "node:path";
561
568
  import { finished } from "node:stream/promises";
@@ -621,7 +628,7 @@ var IndexCommand = class extends SqlCommand {
621
628
  names,
622
629
  onProgress: createProgressHandler(eventsStream, verbose, ctx.stderr)
623
630
  });
624
- await writeFile2(fragmentsPath, JSON.stringify(fragments, null, 2));
631
+ await writeFile(fragmentsPath, JSON.stringify(fragments, null, 2));
625
632
  const manifest = {
626
633
  fragmentsPath,
627
634
  eventsPath,
@@ -734,7 +741,7 @@ function dedupe(values) {
734
741
  }
735
742
 
736
743
  // packages/text2sql/src/bin/commands/run.ts
737
- import { mkdir as mkdir3, writeFile as writeFile3 } from "node:fs/promises";
744
+ import { mkdir as mkdir3, writeFile as writeFile2 } from "node:fs/promises";
738
745
  import * as path4 from "node:path";
739
746
  import { v7 as v72 } from "uuid";
740
747
  var RunCommand = class extends SqlQueryCommand {
@@ -749,7 +756,7 @@ var RunCommand = class extends SqlQueryCommand {
749
756
  const outPath = path4.join(outputDir, `${v72()}.json`);
750
757
  try {
751
758
  await mkdir3(outputDir, { recursive: true });
752
- await writeFile3(outPath, JSON.stringify(rows, null, 2));
759
+ await writeFile2(outPath, JSON.stringify(rows, null, 2));
753
760
  } catch (error) {
754
761
  this.fail(errorMessage2(error));
755
762
  }
@@ -888,7 +895,7 @@ function resolveIndexCache() {
888
895
  if (!dir && !namespace) {
889
896
  return void 0;
890
897
  }
891
- return new FileIndexCache({ dir, namespace });
898
+ return new FileIndexCache({ clock: globalThis, dir, namespace });
892
899
  }
893
900
  async function runCommand(command, positional, options) {
894
901
  let ctx;
@@ -1,18 +1,2 @@
1
- /**
2
- * Convert HuggingFace model to GGUF format using Docker
3
- * Uses the official llama.cpp image - no local installation needed
4
- *
5
- * Usage: node convert-to-gguf.ts <model_dir> <output.gguf> [quantization]
6
- *
7
- * Arguments:
8
- * model_dir Path to HuggingFace model directory (contains model.safetensors)
9
- * output.gguf Output path for the GGUF file
10
- * quantization Quantization type (default: q8_0)
11
- * Options: f32, f16, bf16, q8_0, q4_0, q4_1, q5_0, q5_1
12
- *
13
- * Examples:
14
- * node convert-to-gguf.ts ./qwen3-sql-final ./qwen3-sql.gguf
15
- * node convert-to-gguf.ts ./qwen3-sql-final ./qwen3-sql-q4.gguf q4_0
16
- */
17
1
  export {};
18
2
  //# sourceMappingURL=convert-to-gguf.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"convert-to-gguf.d.ts","sourceRoot":"","sources":["../../src/finetune/convert-to-gguf.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG"}
1
+ {"version":3,"file":"convert-to-gguf.d.ts","sourceRoot":"","sources":["../../src/finetune/convert-to-gguf.ts"],"names":[],"mappings":""}
package/dist/index.js CHANGED
@@ -19,6 +19,7 @@ function validateAdapterNames(names) {
19
19
  }
20
20
 
21
21
  // packages/text2sql/src/lib/adapters/adapter.ts
22
+ import pMemoize from "p-memoize";
22
23
  import { format as formatSql } from "sql-formatter";
23
24
  import { z } from "zod";
24
25
 
@@ -431,12 +432,7 @@ var Adapter = class {
431
432
  #decodeShellEscapes(sql) {
432
433
  return sql.replace(/\\n/g, "\n").replace(/\\t/g, " ").replace(/\\([().*$])/g, "$1").replace(/\\(?=$)/gm, "");
433
434
  }
434
- #cachedAllowedEntities = null;
435
- async #resolveScope() {
436
- if (this.#cachedAllowedEntities) return this.#cachedAllowedEntities;
437
- this.#cachedAllowedEntities = await this.resolveAllowedEntities();
438
- return this.#cachedAllowedEntities;
439
- }
435
+ #resolveScope = pMemoize(() => this.resolveAllowedEntities());
440
436
  async #analyzePolicy(sql) {
441
437
  return this.#policyAnalyzer.analyze(sql, {
442
438
  resolveAllowedEntities: () => this.#resolveScope()
@@ -1129,16 +1125,19 @@ import {
1129
1125
  } from "@deepagents/context";
1130
1126
 
1131
1127
  // packages/text2sql/src/lib/file-cache.ts
1132
- import { createHash, randomUUID } from "node:crypto";
1128
+ import { atomicWrite } from "@zukhruf/fs";
1129
+ import { createHash } from "node:crypto";
1133
1130
  import { existsSync } from "node:fs";
1134
- import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
1131
+ import { mkdir, readFile } from "node:fs/promises";
1135
1132
  import { tmpdir } from "node:os";
1136
1133
  import path from "node:path";
1137
1134
  var FileCache = class {
1138
1135
  path;
1139
- constructor(watermark, extension = ".txt", baseDir = tmpdir()) {
1136
+ #clock;
1137
+ constructor(watermark, { clock, extension = ".txt", baseDir = tmpdir() }) {
1140
1138
  const hash2 = createHash("md5").update(watermark).digest("hex");
1141
1139
  this.path = path.join(baseDir, `text2sql-${hash2}${extension}`);
1140
+ this.#clock = clock;
1142
1141
  }
1143
1142
  async get() {
1144
1143
  if (existsSync(this.path)) {
@@ -1148,9 +1147,7 @@ var FileCache = class {
1148
1147
  }
1149
1148
  async set(content) {
1150
1149
  await mkdir(path.dirname(this.path), { recursive: true });
1151
- const tempPath = `${this.path}.${randomUUID()}.tmp`;
1152
- await writeFile(tempPath, content, "utf-8");
1153
- await rename(tempPath, this.path);
1150
+ await atomicWrite(this.path, content, { clock: this.#clock });
1154
1151
  }
1155
1152
  };
1156
1153
  var JsonCache = class extends FileCache {
@@ -1159,8 +1156,8 @@ var JsonCache = class extends FileCache {
1159
1156
  * @param isValid - Checks that a parsed cache file holds a T; a file that
1160
1157
  * fails it is read as a miss.
1161
1158
  */
1162
- constructor(watermark, isValid, baseDir) {
1163
- super(watermark, ".json", baseDir);
1159
+ constructor(watermark, isValid, { clock, baseDir }) {
1160
+ super(watermark, { clock, extension: ".json", baseDir });
1164
1161
  this.#isValid = isValid;
1165
1162
  }
1166
1163
  async read() {
@@ -1182,9 +1179,11 @@ var JsonCache = class extends FileCache {
1182
1179
 
1183
1180
  // packages/text2sql/src/lib/index-cache.ts
1184
1181
  var FileIndexCache = class {
1182
+ #clock;
1185
1183
  #dir;
1186
1184
  #namespace;
1187
- constructor(options = {}) {
1185
+ constructor(options) {
1186
+ this.#clock = options.clock;
1188
1187
  this.#dir = options.dir;
1189
1188
  this.#namespace = options.namespace;
1190
1189
  }
@@ -1196,7 +1195,10 @@ var FileIndexCache = class {
1196
1195
  }
1197
1196
  #cacheFor(key) {
1198
1197
  const watermark = this.#namespace ? `${this.#namespace}-${key}` : key;
1199
- return new JsonCache(watermark, isFragmentList, this.#dir);
1198
+ return new JsonCache(watermark, isFragmentList, {
1199
+ clock: this.#clock,
1200
+ baseDir: this.#dir
1201
+ });
1200
1202
  }
1201
1203
  };
1202
1204
  function isFragmentList(value) {
@@ -1367,8 +1369,9 @@ function errorMessage(error) {
1367
1369
  }
1368
1370
 
1369
1371
  // packages/text2sql/src/lib/checkpoint.ts
1372
+ import { durableWrite } from "@zukhruf/fs";
1370
1373
  import { createHash as createHash2 } from "node:crypto";
1371
- import { existsSync as existsSync2, readFileSync, renameSync, writeFileSync } from "node:fs";
1374
+ import { existsSync as existsSync2, readFileSync } from "node:fs";
1372
1375
  import pLimit from "p-limit";
1373
1376
  import z4 from "zod";
1374
1377
  function jsonCodec(schema) {
@@ -1388,24 +1391,29 @@ var Checkpoint = class _Checkpoint {
1388
1391
  points;
1389
1392
  path;
1390
1393
  configHash;
1391
- constructor(path6, configHash, points) {
1394
+ #clock;
1395
+ #writes = pLimit(1);
1396
+ #changes = 0;
1397
+ #writtenChanges = 0;
1398
+ constructor(path6, configHash, clock, points) {
1392
1399
  this.points = points;
1393
1400
  this.path = path6;
1394
1401
  this.configHash = configHash;
1402
+ this.#clock = clock;
1395
1403
  }
1396
1404
  /**
1397
1405
  * Load checkpoint from file, or return empty checkpoint if none exists.
1398
1406
  * Handles corrupted files and config changes gracefully.
1399
1407
  */
1400
1408
  static async load(options) {
1401
- const { path: path6, configHash } = options;
1409
+ const { path: path6, configHash, clock } = options;
1402
1410
  if (existsSync2(path6)) {
1403
1411
  try {
1404
1412
  const content = readFileSync(path6, "utf-8");
1405
1413
  const file = checkpointFile.parse(JSON.parse(content));
1406
1414
  if (configHash && file.configHash && file.configHash !== configHash) {
1407
1415
  console.log("\u26A0 Config changed, starting fresh");
1408
- return new _Checkpoint(path6, configHash, {});
1416
+ return new _Checkpoint(path6, configHash, clock, {});
1409
1417
  }
1410
1418
  const points = file.points;
1411
1419
  const totalEntries = Object.values(points).reduce(
@@ -1413,14 +1421,14 @@ var Checkpoint = class _Checkpoint {
1413
1421
  0
1414
1422
  );
1415
1423
  console.log(`\u2713 Resuming from checkpoint (${totalEntries} entries)`);
1416
- return new _Checkpoint(path6, configHash, points);
1424
+ return new _Checkpoint(path6, configHash, clock, points);
1417
1425
  } catch {
1418
1426
  console.log("\u26A0 Checkpoint corrupted, starting fresh");
1419
- return new _Checkpoint(path6, configHash, {});
1427
+ return new _Checkpoint(path6, configHash, clock, {});
1420
1428
  }
1421
1429
  }
1422
1430
  console.log("Starting new checkpoint");
1423
- return new _Checkpoint(path6, configHash, {});
1431
+ return new _Checkpoint(path6, configHash, clock, {});
1424
1432
  }
1425
1433
  /**
1426
1434
  * Run a single computation with checkpointing.
@@ -1484,15 +1492,25 @@ var Checkpoint = class _Checkpoint {
1484
1492
  getPath() {
1485
1493
  return this.path;
1486
1494
  }
1487
- async save() {
1488
- const file = {
1489
- configHash: this.configHash,
1490
- points: this.points
1491
- };
1492
- const content = JSON.stringify(file, null, 2);
1493
- const tempPath = `${this.path}.tmp`;
1494
- writeFileSync(tempPath, content);
1495
- renameSync(tempPath, this.path);
1495
+ /**
1496
+ * Writes one snapshot at a time, so an older snapshot never lands after a
1497
+ * newer one. Each write takes the snapshot when it starts, so a save that
1498
+ * waited behind a write that already holds its change writes nothing.
1499
+ */
1500
+ save() {
1501
+ this.#changes += 1;
1502
+ return this.#writes(async () => {
1503
+ const changes = this.#changes;
1504
+ if (changes === this.#writtenChanges) return;
1505
+ const file = {
1506
+ configHash: this.configHash,
1507
+ points: this.points
1508
+ };
1509
+ await durableWrite(this.path, JSON.stringify(file, null, 2), {
1510
+ clock: this.#clock
1511
+ });
1512
+ this.#writtenChanges = changes;
1513
+ });
1496
1514
  }
1497
1515
  };
1498
1516
  function hash(value) {
@@ -1546,9 +1564,9 @@ function hashConfig(config) {
1546
1564
 
1547
1565
  // packages/text2sql/src/lib/fs/sqlite/sqlite-fs.ts
1548
1566
  import { Declaration } from "@zukhruf/sqlite-drift";
1567
+ import { open } from "@zukhruf/sqlite-open";
1549
1568
  import { transaction } from "@zukhruf/sqlite-transaction";
1550
1569
  import * as path2 from "node:path";
1551
- import { DatabaseSync } from "node:sqlite";
1552
1570
  import z6 from "zod";
1553
1571
 
1554
1572
  // packages/text2sql/src/lib/fs/columns.ts
@@ -1558,7 +1576,6 @@ var bigintColumn = z5.union([z5.number(), z5.string().regex(/^-?\d+$/)]).transfo
1558
1576
 
1559
1577
  // packages/text2sql/src/lib/fs/sqlite/ddl.sqlite-fs.ts
1560
1578
  var SETUP = `
1561
- PRAGMA journal_mode = WAL;
1562
1579
  PRAGMA synchronous = NORMAL;
1563
1580
  PRAGMA cache_size = -64000;
1564
1581
  PRAGMA temp_store = MEMORY;
@@ -1620,7 +1637,7 @@ var SqliteFs = class {
1620
1637
  this.#chunkSize = options.chunkSize ?? 1024 * 1024;
1621
1638
  const normalizedRoot = this.#normalizeRoot(options.root);
1622
1639
  this.#root = normalizedRoot === "/" ? "" : normalizedRoot;
1623
- this.#db = new DatabaseSync(options.dbPath);
1640
+ this.#db = open(options.dbPath);
1624
1641
  this.#db.exec(SETUP);
1625
1642
  tables.apply(this.#db);
1626
1643
  this.#sql = this.#db.createTagStore();