@deepagents/text2sql 2.3.0 → 3.0.0

package/README.md

@@ -67,7 +67,6 @@ const adapter = new Postgres({
 });
 
 const text2sql = new Text2Sql({
-  version: 'v1',
   model: groq('openai/gpt-oss-20b'),
   adapters: { main: adapter },
 });
@@ -194,8 +193,12 @@ sandbox was not prepared correctly.
 - `--out-dir <path>`: writes artifacts under that path (default:
   `$TEXT2SQL_OUT_DIR` or `./sql`).
 
-Set `TEXT2SQL_INDEX_VERSION` to manage cache invalidation across runs. Cache
-keys are `index-<version>-<adapter>`, so bump the version when schema changes.
+The `sql` CLI caches introspected schema only when you opt in via env: set
+`TEXT2SQL_INDEX_CACHE_DIR` (where cache files live) and/or
+`TEXT2SQL_INDEX_VERSION` (an invalidation token — bump it when the schema
+changes). With neither set, every `sql index` introspects fresh. See
+[Schema index caching & coordination](#schema-index-caching--coordination) for
+the underlying injectable primitives.
 
 For in-process or virtual-sandbox usage (without installing the package CLI),
 wrap an existing `Text2Sql` instance as a just-bash custom command:
@@ -328,6 +331,69 @@ for await (const chunk of followUp) {
 }
 ```
 
+## Schema index caching & coordination
+
+Schema introspection is the expensive part of indexing. `Text2Sql` and
+`AdapterIndexer` own no storage and no locking — you inject both as optional
+primitives, keyed per adapter:
+
+```typescript
+export interface IndexCache {
+  read(key: string): Promise<ContextFragment[] | null>;
+  write(key: string, fragments: ContextFragment[]): Promise<void>;
+}
+
+export interface IndexLock {
+  // Acquire (waiting if held elsewhere), run fn, release — even on throw.
+  // Reject if the lock can't be acquired (callers fail closed).
+  run<T>(key: string, fn: () => Promise<T>): Promise<T>;
+}
+```
+
+- **No `cache`** → every `index()` introspects fresh (no cache events).
+- **`cache` only** → warm reads skip introspection within whatever the cache
+  spans.
+- **`cache` + `lock`** → introspection is single-flight: concurrent callers for
+  the same adapter wait on the lock, then read the warm cache.
+
+A file-backed `IndexCache` ships as a composable primitive:
+
+```typescript
+import { FileIndexCache, Text2Sql } from '@deepagents/text2sql';
+
+const text2sql = new Text2Sql({
+  model,
+  adapters: { main: adapter },
+  // Point `dir` at a shared volume to share one cache across processes.
+  cache: new FileIndexCache({ dir: '/var/cache/text2sql', namespace: 'v1' }),
+});
+```
+
+`FileIndexCache` writes atomically (temp + rename) and treats an unparseable
+file as a miss, so a torn read self-heals into a re-introspect.
+
+### Horizontally-scaled deployments
+
+Running many processes/containers against one database (e.g. a fleet of
+daemons)? Two coordination tiers:
+
+- The **lock alone** serializes introspection — never two concurrent
+  introspections of the same database (caps peak DB load). Back it with a
+  distributed lock you already operate (a Redis lock, a Postgres
+  `pg_advisory_lock`, etc.).
+- The **lock plus a shared cache directory** gives fleet-wide single-flight:
+  the lock holder writes the cache on the shared volume, and every waiter reads
+  it instead of re-introspecting.
+
+> On object-storage-backed volumes (GCS/S3 FUSE) `rename` is not atomic and
+> file locks are unreliable — that is exactly why the lock and cache are
+> injectable. Use a real distributed lock; the `FileIndexCache` parse-as-miss
+> behavior plus the post-lock recheck tolerate a non-atomic write.
+
+See `demo/text2sql-daemon` for a daemon that injects a `FileIndexCache`
+(`TEXT2SQL_INDEX_CACHE_DIR` / `TEXT2SQL_INDEX_VERSION`) and a `pg_advisory_lock`
+backed `IndexLock`.
+
 ## Streaming Index Progress
 
 `AdapterIndexer#index({ onProgress })` emits progress events while it warms or reads
@@ -343,7 +409,7 @@ import {
   type Text2SqlIndexProgressEvent,
 } from '@deepagents/text2sql';
 
-const indexer = new AdapterIndexer({ adapters, version: 'v1' });
+const indexer = new AdapterIndexer({ adapters });
 
 await context.continue(user('Show me top 10 customers'));
 

package/dist/bin/sql.js

@@ -3,6 +3,71 @@
 // packages/text2sql/src/bin/sql.ts
 import { cac } from "cac";
 
+// packages/text2sql/src/lib/file-cache.ts
+import { createHash, randomUUID } from "node:crypto";
+import { existsSync } from "node:fs";
+import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+var FileCache = class {
+  path;
+  constructor(watermark, extension = ".txt", baseDir = tmpdir()) {
+    const hash = createHash("md5").update(watermark).digest("hex");
+    this.path = path.join(baseDir, `text2sql-${hash}${extension}`);
+  }
+  async get() {
+    if (existsSync(this.path)) {
+      return readFile(this.path, "utf-8");
+    }
+    return null;
+  }
+  async set(content) {
+    await mkdir(path.dirname(this.path), { recursive: true });
+    const tempPath = `${this.path}.${randomUUID()}.tmp`;
+    await writeFile(tempPath, content, "utf-8");
+    await rename(tempPath, this.path);
+  }
+};
+var JsonCache = class extends FileCache {
+  constructor(watermark, baseDir) {
+    super(watermark, ".json", baseDir);
+  }
+  async read() {
+    const content = await this.get();
+    if (!content) {
+      return null;
+    }
+    try {
+      return JSON.parse(content);
+    } catch {
+      return null;
+    }
+  }
+  write(data) {
+    return this.set(JSON.stringify(data));
+  }
+};
+
+// packages/text2sql/src/lib/index-cache.ts
+var FileIndexCache = class {
+  #dir;
+  #namespace;
+  constructor(options = {}) {
+    this.#dir = options.dir;
+    this.#namespace = options.namespace;
+  }
+  read(key) {
+    return this.#cacheFor(key).read();
+  }
+  write(key, fragments) {
+    return this.#cacheFor(key).write(fragments);
+  }
+  #cacheFor(key) {
+    const watermark = this.#namespace ? `${this.#namespace}-${key}` : key;
+    return new JsonCache(watermark, this.#dir);
+  }
+};
+
 // packages/text2sql/src/lib/sql.ts
 import "@deepagents/agent";
 
@@ -120,44 +185,6 @@ function buildScopeParseErrorPayload(sql, dialect, error) {
   };
 }
 
-// packages/text2sql/src/lib/file-cache.ts
-import { createHash } from "node:crypto";
-import { existsSync } from "node:fs";
-import { readFile, writeFile } from "node:fs/promises";
-import { tmpdir } from "node:os";
-import path from "node:path";
-var FileCache = class {
-  path;
-  constructor(watermark, extension = ".txt") {
-    const hash = createHash("md5").update(watermark).digest("hex");
-    this.path = path.join(tmpdir(), `text2sql-${hash}${extension}`);
-  }
-  async get() {
-    if (existsSync(this.path)) {
-      return readFile(this.path, "utf-8");
-    }
-    return null;
-  }
-  set(content) {
-    return writeFile(this.path, content, "utf-8");
-  }
-};
-var JsonCache = class extends FileCache {
-  constructor(watermark) {
-    super(watermark, ".json");
-  }
-  async read() {
-    const content = await this.get();
-    if (content) {
-      return JSON.parse(content);
-    }
-    return null;
-  }
-  write(data) {
-    return this.set(JSON.stringify(data));
-  }
-};
-
 // packages/text2sql/src/lib/adapter-index.ts
 var noopProgressHandler = () => {
 };
@@ -172,8 +199,8 @@ function timestampProgressHandler(onProgress) {
 }
 var AdapterIndexer = class {
   #adapters;
-  #version;
-  #cacheKey;
+  #cache;
+  #lock;
   constructor(options) {
     const adapterNames = Object.keys(options.adapters);
     if (adapterNames.length === 0) {
@@ -181,8 +208,8 @@ var AdapterIndexer = class {
     }
     validateAdapterNames(adapterNames);
     this.#adapters = options.adapters;
-    this.#version = options.version;
-    this.#cacheKey = options.cacheKey;
+    this.#cache = options.cache;
+    this.#lock = options.lock;
   }
   async index(options = {}) {
     const progress = timestampProgressHandler(options.onProgress);
@@ -239,17 +266,42 @@ var AdapterIndexer = class {
       adapter: name,
       message: `Indexing adapter "${name}"...`
     });
-    const cacheKey = this.#adapterCacheKey(name);
-    const cache = cacheKey ? new JsonCache(cacheKey) : void 0;
+    const cache = this.#cache;
+    const key = name;
+    const emitCacheHit = () => progress({
+      type: "adapter:cache-hit",
+      adapter: name,
+      message: `Using cached index for adapter "${name}".`,
+      cached: true
+    });
+    const emitCacheMiss = () => progress({
+      type: "adapter:cache-miss",
+      adapter: name,
+      message: `No cached index for adapter "${name}".`,
+      cached: false
+    });
+    const introspect = async () => {
+      const ctx = createGroundingContext({
+        onProgress: (event) => progress(adapterProgressEvent(name, event))
+      });
+      const fragments = await adapter.introspect(ctx);
+      await cache?.write(key, fragments);
+      return fragments;
+    };
+    let servedFromCache = false;
+    const populate = async () => {
+      const recheck = await cache?.read(key);
+      if (recheck) {
+        servedFromCache = true;
+        return recheck;
+      }
+      if (cache) emitCacheMiss();
+      return introspect();
+    };
     try {
-      const cached = await cache?.read();
+      const cached = await cache?.read(key);
       if (cached) {
-        progress({
-          type: "adapter:cache-hit",
-          adapter: name,
-          message: `Using cached index for adapter "${name}".`,
-          cached: true
-        });
+        emitCacheHit();
         progress({
           type: "adapter:end",
           adapter: name,
@@ -258,24 +310,13 @@ var AdapterIndexer = class {
         });
         return cached;
       }
-      if (cache) {
-        progress({
-          type: "adapter:cache-miss",
-          adapter: name,
-          message: `No cached index for adapter "${name}".`,
-          cached: false
-        });
-      }
-      const ctx = createGroundingContext({
-        onProgress: (event) => progress(adapterProgressEvent(name, event))
-      });
-      const fragments = await adapter.introspect(ctx);
-      await cache?.write(fragments);
+      const fragments = this.#lock ? await this.#lock.run(key, populate) : await populate();
+      if (servedFromCache) emitCacheHit();
       progress({
         type: "adapter:end",
         adapter: name,
         message: `Finished indexing adapter "${name}".`,
-        cached: false
+        cached: servedFromCache
       });
       return fragments;
     } catch (error) {
@@ -302,16 +343,6 @@ var AdapterIndexer = class {
       return [name, adapter];
     });
   }
-  #adapterCacheKey(name) {
-    const configured = this.#cacheKey?.(name);
-    if (configured !== void 0) {
-      return configured;
-    }
-    if (this.#version) {
-      return `index-${this.#version}-${name}`;
-    }
-    return void 0;
-  }
 };
 function adapterProgressEvent(adapter, progress) {
   return {
@@ -824,7 +855,8 @@ var Text2Sql = class {
     this.#config = config;
     this.#indexer = new AdapterIndexer({
       adapters: config.adapters,
-      version: config.version
+      cache: config.cache,
+      lock: config.lock
     });
   }
   #requireAdapter(name) {
@@ -959,7 +991,7 @@ var SqlQueryCommand = class extends SqlCommand {
 // packages/text2sql/src/bin/commands/index.ts
 import { constants, createWriteStream } from "node:fs";
 import {
-  mkdir,
+  mkdir as mkdir2,
   open,
   stat,
   writeFile as writeFile2
@@ -1012,9 +1044,9 @@ var IndexCommand = class extends SqlCommand {
     const explicitEventsPath = resolveExplicitEventsPath(ctx);
     const eventsPath = explicitEventsPath ?? path3.join(outputDir, `index-${id}.events.ndjson`);
     try {
-      await mkdir(outputDir, { recursive: true });
+      await mkdir2(outputDir, { recursive: true });
       if (explicitEventsPath) {
-        await mkdir(path3.dirname(explicitEventsPath), { recursive: true });
+        await mkdir2(path3.dirname(explicitEventsPath), { recursive: true });
       }
       const eventsStream = await openEventsWriter(
         eventsPath,
@@ -1145,7 +1177,7 @@ function dedupe(values) {
 }
 
 // packages/text2sql/src/bin/commands/run.ts
-import { mkdir as mkdir2, writeFile as writeFile3 } from "node:fs/promises";
+import { mkdir as mkdir3, writeFile as writeFile3 } from "node:fs/promises";
 import * as path4 from "node:path";
 import { v7 as v72 } from "uuid";
 var RunCommand = class extends SqlQueryCommand {
@@ -1159,7 +1191,7 @@ var RunCommand = class extends SqlQueryCommand {
     const { rows, columns } = await ctx.text2Sql.run(name, sql);
     const outPath = path4.join(outputDir, `${v72()}.json`);
     try {
-      await mkdir2(outputDir, { recursive: true });
+      await mkdir3(outputDir, { recursive: true });
       await writeFile3(outPath, JSON.stringify(rows, null, 2));
     } catch (error) {
       this.fail(errorMessage2(error));
@@ -1296,12 +1328,20 @@ function registerCommand(cli2, command) {
     return runCommand(command, positional, options);
   });
 }
+function resolveIndexCache() {
+  const dir = process.env.TEXT2SQL_INDEX_CACHE_DIR;
+  const namespace = process.env.TEXT2SQL_INDEX_VERSION;
+  if (!dir && !namespace) {
+    return void 0;
+  }
+  return new FileIndexCache({ dir, namespace });
+}
 async function runCommand(command, positional, options) {
   let ctx;
   try {
     const text2Sql = new Text2Sql({
       adapters: await loadAdapters(),
-      version: process.env.TEXT2SQL_INDEX_VERSION
+      cache: resolveIndexCache()
     });
     ctx = {
       text2Sql,

package/dist/index.d.ts

@@ -6,6 +6,8 @@ export * from './lib/agents/suggestions.agents.ts';
 export * from './lib/adapter-index.ts';
 export * from './lib/checkpoint.ts';
 export * from './lib/file-cache.ts';
+export * from './lib/index-cache.ts';
+export * from './lib/index-lock.ts';
 export * from './lib/fragments/schema.ts';
 export * from './lib/fs/index.ts';
 export * from './lib/instructions.ts';

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,kBAAkB,EAClB,oBAAoB,GACrB,MAAM,uBAAuB,CAAC;AAC/B,cAAc,2BAA2B,CAAC;AAC1C,cAAc,4BAA4B,CAAC;AAC3C,cAAc,2BAA2B,CAAC;AAC1C,cAAc,oCAAoC,CAAC;AACnD,cAAc,wBAAwB,CAAC;AACvC,cAAc,qBAAqB,CAAC;AACpC,cAAc,qBAAqB,CAAC;AACpC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,mBAAmB,CAAC;AAClC,cAAc,uBAAuB,CAAC;AACtC,cAAc,oBAAoB,CAAC;AACnC,OAAO,EACL,KAAK,uBAAuB,EAC5B,KAAK,sBAAsB,EAC3B,gBAAgB,GACjB,MAAM,sBAAsB,CAAC;AAC9B,cAAc,cAAc,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,kBAAkB,EAClB,oBAAoB,GACrB,MAAM,uBAAuB,CAAC;AAC/B,cAAc,2BAA2B,CAAC;AAC1C,cAAc,4BAA4B,CAAC;AAC3C,cAAc,2BAA2B,CAAC;AAC1C,cAAc,oCAAoC,CAAC;AACnD,cAAc,wBAAwB,CAAC;AACvC,cAAc,qBAAqB,CAAC;AACpC,cAAc,qBAAqB,CAAC;AACpC,cAAc,sBAAsB,CAAC;AACrC,cAAc,qBAAqB,CAAC;AACpC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,mBAAmB,CAAC;AAClC,cAAc,uBAAuB,CAAC;AACtC,cAAc,oBAAoB,CAAC;AACnC,OAAO,EACL,KAAK,uBAAuB,EAC5B,KAAK,sBAAsB,EAC3B,gBAAgB,GACjB,MAAM,sBAAsB,CAAC;AAC9B,cAAc,cAAc,CAAC"}

package/dist/index.js

@@ -1301,16 +1301,16 @@ var suggestionsAgent = agent({
 import { fragment as fragment2 } from "@deepagents/context";
 
 // packages/text2sql/src/lib/file-cache.ts
-import { createHash } from "node:crypto";
+import { createHash, randomUUID } from "node:crypto";
 import { existsSync } from "node:fs";
-import { readFile, writeFile } from "node:fs/promises";
+import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import path from "node:path";
 var FileCache = class {
   path;
-  constructor(watermark, extension = ".txt") {
+  constructor(watermark, extension = ".txt", baseDir = tmpdir()) {
     const hash2 = createHash("md5").update(watermark).digest("hex");
-    this.path = path.join(tmpdir(), `text2sql-${hash2}${extension}`);
+    this.path = path.join(baseDir, `text2sql-${hash2}${extension}`);
   }
   async get() {
     if (existsSync(this.path)) {
@@ -1318,26 +1318,53 @@ var FileCache = class {
     }
     return null;
   }
-  set(content) {
-    return writeFile(this.path, content, "utf-8");
+  async set(content) {
+    await mkdir(path.dirname(this.path), { recursive: true });
+    const tempPath = `${this.path}.${randomUUID()}.tmp`;
+    await writeFile(tempPath, content, "utf-8");
+    await rename(tempPath, this.path);
   }
 };
 var JsonCache = class extends FileCache {
-  constructor(watermark) {
-    super(watermark, ".json");
+  constructor(watermark, baseDir) {
+    super(watermark, ".json", baseDir);
   }
   async read() {
     const content = await this.get();
-    if (content) {
+    if (!content) {
+      return null;
+    }
+    try {
       return JSON.parse(content);
+    } catch {
+      return null;
     }
-    return null;
   }
   write(data) {
     return this.set(JSON.stringify(data));
   }
 };
 
+// packages/text2sql/src/lib/index-cache.ts
+var FileIndexCache = class {
+  #dir;
+  #namespace;
+  constructor(options = {}) {
+    this.#dir = options.dir;
+    this.#namespace = options.namespace;
+  }
+  read(key) {
+    return this.#cacheFor(key).read();
+  }
+  write(key, fragments) {
+    return this.#cacheFor(key).write(fragments);
+  }
+  #cacheFor(key) {
+    const watermark = this.#namespace ? `${this.#namespace}-${key}` : key;
+    return new JsonCache(watermark, this.#dir);
+  }
+};
+
 // packages/text2sql/src/lib/adapter-index.ts
 var TEXT2SQL_INDEX_PROGRESS_CHUNK = "data-text2sql-index-progress";
 var noopProgressHandler = () => {
@@ -1353,8 +1380,8 @@ function timestampProgressHandler(onProgress) {
 }
 var AdapterIndexer = class {
   #adapters;
-  #version;
-  #cacheKey;
+  #cache;
+  #lock;
   constructor(options) {
     const adapterNames = Object.keys(options.adapters);
     if (adapterNames.length === 0) {
@@ -1362,8 +1389,8 @@ var AdapterIndexer = class {
     }
     validateAdapterNames(adapterNames);
     this.#adapters = options.adapters;
-    this.#version = options.version;
-    this.#cacheKey = options.cacheKey;
+    this.#cache = options.cache;
+    this.#lock = options.lock;
   }
   async index(options = {}) {
     const progress = timestampProgressHandler(options.onProgress);
@@ -1420,17 +1447,42 @@ var AdapterIndexer = class {
       adapter: name,
       message: `Indexing adapter "${name}"...`
     });
-    const cacheKey = this.#adapterCacheKey(name);
-    const cache = cacheKey ? new JsonCache(cacheKey) : void 0;
+    const cache = this.#cache;
+    const key = name;
+    const emitCacheHit = () => progress({
+      type: "adapter:cache-hit",
+      adapter: name,
+      message: `Using cached index for adapter "${name}".`,
+      cached: true
+    });
+    const emitCacheMiss = () => progress({
+      type: "adapter:cache-miss",
+      adapter: name,
+      message: `No cached index for adapter "${name}".`,
+      cached: false
+    });
+    const introspect = async () => {
+      const ctx = createGroundingContext({
+        onProgress: (event) => progress(adapterProgressEvent(name, event))
+      });
+      const fragments = await adapter.introspect(ctx);
+      await cache?.write(key, fragments);
+      return fragments;
+    };
+    let servedFromCache = false;
+    const populate = async () => {
+      const recheck = await cache?.read(key);
+      if (recheck) {
+        servedFromCache = true;
+        return recheck;
+      }
+      if (cache) emitCacheMiss();
+      return introspect();
+    };
     try {
-      const cached = await cache?.read();
+      const cached = await cache?.read(key);
       if (cached) {
-        progress({
-          type: "adapter:cache-hit",
-          adapter: name,
-          message: `Using cached index for adapter "${name}".`,
-          cached: true
-        });
+        emitCacheHit();
         progress({
           type: "adapter:end",
           adapter: name,
@@ -1439,24 +1491,13 @@ var AdapterIndexer = class {
         });
         return cached;
       }
-      if (cache) {
-        progress({
-          type: "adapter:cache-miss",
-          adapter: name,
-          message: `No cached index for adapter "${name}".`,
-          cached: false
-        });
-      }
-      const ctx = createGroundingContext({
-        onProgress: (event) => progress(adapterProgressEvent(name, event))
-      });
-      const fragments = await adapter.introspect(ctx);
-      await cache?.write(fragments);
+      const fragments = this.#lock ? await this.#lock.run(key, populate) : await populate();
+      if (servedFromCache) emitCacheHit();
       progress({
         type: "adapter:end",
         adapter: name,
         message: `Finished indexing adapter "${name}".`,
-        cached: false
+        cached: servedFromCache
       });
       return fragments;
     } catch (error) {
@@ -1483,16 +1524,6 @@ var AdapterIndexer = class {
       return [name, adapter];
     });
   }
-  #adapterCacheKey(name) {
-    const configured = this.#cacheKey?.(name);
-    if (configured !== void 0) {
-      return configured;
-    }
-    if (this.#version) {
-      return `index-${this.#version}-${name}`;
-    }
-    return void 0;
-  }
 };
 function adapterProgressEvent(adapter, progress) {
   return {
@@ -5580,7 +5611,8 @@ var Text2Sql = class {
     this.#config = config;
     this.#indexer = new AdapterIndexer({
       adapters: config.adapters,
-      version: config.version
+      cache: config.cache,
+      lock: config.lock
     });
   }
   #requireAdapter(name) {
@@ -5790,6 +5822,7 @@ export {
   AdapterIndexer,
   Checkpoint,
   FileCache,
+  FileIndexCache,
   JsonCache,
   MssqlFs,
   Point,