@deepagents/text2sql 3.0.1 → 4.0.0

package/README.md

@@ -34,7 +34,7 @@ Requires Node.js LTS
 import { groq } from '@ai-sdk/groq';
 import pg from 'pg';
 
-import { Text2Sql } from '@deepagents/text2sql';
+import { FileIndexLock, Text2Sql } from '@deepagents/text2sql';
 import {
   Postgres,
   columnValues,
@@ -69,6 +69,7 @@ const adapter = new Postgres({
 const text2sql = new Text2Sql({
   model: groq('openai/gpt-oss-20b'),
   adapters: { main: adapter },
+  lock: new FileIndexLock(),
 });
 
 // Generate SQL
@@ -82,6 +83,8 @@ console.log(sql);
 The adapter-map key (`main` here) is the adapter name. Reuse that same key in
 `text2sql.toSql(..., 'main')` and in any `sql validate <db> "..."` /
 `sql run <db> "..."` calls from a sandbox where the package CLI is installed.
+This is the configured connection name, not the SQL-level database or schema
+name (for example, SQLite's default `main` schema).
 
 Adapter names must match `/^[A-Za-z_][A-Za-z0-9_]*$/`. If you build adapter
 maps dynamically, use `isValidAdapterName(name)` to check one key or
@@ -170,15 +173,21 @@ or upload/write that module before you call `sql index` or `chat()`.
 
 `instructions()` returns the SQL-flavored system fragments (policies, workflows,
 clarifications, style guides) — spread them into `context.set()` alongside the
-schema fragments returned by `sql index`. Add or replace them with your
-own domain fragments as needed.
+schema fragments returned by `sql index`. The index output starts with an
+`available_databases` fragment that lists the exact configured adapter names the
+model must pass to `sql validate <db>` and `sql run <db>`. Add or replace the
+instructions with your own domain fragments as needed.
 
 ## Advanced: SQL CLI in Sandboxes
 
 `sql validate <db> "..."` and `sql run <db> "..."` are real commands from the
-`@deepagents/text2sql` package. `sql index` writes schema fragments plus
-progress events for chat setup, indexing all configured adapters by default
-(same as `--all`) unless adapter names are provided.
+`@deepagents/text2sql` package. `<db>` is the configured adapter name. With a
+single configured adapter, a mistaken database selector is routed to that sole
+adapter and the command prints a note; with multiple adapters, unknown names
+fail and print the available list. `sql index` writes an `available_databases`
+fragment, schema fragments, and progress events for chat setup, indexing all
+configured adapters by default (same as `--all`) unless adapter names are
+provided.
 
 Install the package inside the sandbox and set `TEXT2SQL_ADAPTERS` to a module
 whose default export is `Record<string, Adapter>`. Missing `sql` means the
@@ -208,11 +217,16 @@ import { InMemoryFs, createVirtualSandbox } from '@deepagents/context';
 import {
   type CreateSqlCommandOptions,
   type CreateSqlCommandResult,
+  FileIndexLock,
   Text2Sql,
   createSqlCommand,
 } from '@deepagents/text2sql';
 
-const text2sql = new Text2Sql({ model, adapters: { main: adapter } });
+const text2sql = new Text2Sql({
+  model,
+  adapters: { main: adapter },
+  lock: new FileIndexLock(),
+});
 
 const commandOptions: CreateSqlCommandOptions = {
   outputDir: '/sql-artifacts',
@@ -334,8 +348,8 @@ for await (const chunk of followUp) {
 ## Schema index caching & coordination
 
 Schema introspection is the expensive part of indexing. `Text2Sql` and
-`AdapterIndexer` own no storage and no locking — you inject both as optional
-primitives, keyed per adapter:
+`AdapterIndexer` own no storage and no locking — you inject them, keyed per
+adapter. A `lock` is **required**; a `cache` is optional:
 
 ```typescript
 export interface IndexCache {
@@ -350,27 +364,35 @@ export interface IndexLock {
 }
 ```
 
-- **No `cache`** → every `index()` introspects fresh (no cache events).
-- **`cache` only** → warm reads skip introspection within whatever the cache
-  spans.
-- **`cache` + `lock`** → introspection is single-flight: concurrent callers for
-  the same adapter wait on the lock, then read the warm cache.
+- **`lock` only** (no `cache`) → introspection is serialized but not
+  deduplicated: each waiter re-introspects, because the cache recheck under the
+  lock is what turns serialization into single-flight.
+- **`lock` + `cache`** → introspection is single-flight: concurrent callers for
+  the same adapter wait on the lock, then read the warm cache the holder wrote.
 
-A file-backed `IndexCache` ships as a composable primitive:
+File-backed `IndexCache` and `IndexLock` implementations ship as composable
+primitives:
 
 ```typescript
-import { FileIndexCache, Text2Sql } from '@deepagents/text2sql';
+import { FileIndexCache, FileIndexLock, Text2Sql } from '@deepagents/text2sql';
 
 const text2sql = new Text2Sql({
   model,
   adapters: { main: adapter },
-  // Point `dir` at a shared volume to share one cache across processes.
+  // Point both `dir`s at a shared volume to share one cache + lock across
+  // processes on the same filesystem.
   cache: new FileIndexCache({ dir: '/var/cache/text2sql', namespace: 'v1' }),
+  lock: new FileIndexLock({ dir: '/var/cache/text2sql', namespace: 'v1' }),
 });
 ```
 
 `FileIndexCache` writes atomically (temp + rename) and treats an unparseable
-file as a miss, so a torn read self-heals into a re-introspect.
+file as a miss, so a torn read self-heals into a re-introspect. `FileIndexLock`
+(built on `proper-lockfile`, options `{ dir?, namespace?, stale?, retries? }`)
+serializes processes that share a POSIX filesystem; a held lock auto-refreshes
+its mtime so a slow introspection is not mistaken for a crash, and acquisition
+fails closed once retries are exhausted. Both `dir`s default to the OS temp
+directory.
 
 ### Horizontally-scaled deployments
 
@@ -378,8 +400,10 @@ Running many processes/containers against one database (e.g. a fleet of
 daemons)? Two coordination tiers:
 
 - The **lock alone** serializes introspection — never two concurrent
-  introspections of the same database (caps peak DB load). Back it with a
-  distributed lock you already operate (a Redis lock, a Postgres
+  introspections of the same database (caps peak DB load). The shipped
+  `FileIndexLock` covers processes that share a POSIX filesystem (one host, or a
+  shared volume — NFSv4 / EFS); for hosts without a shared filesystem, back it
+  with a distributed lock you already operate (a Redis lock, a Postgres
   `pg_advisory_lock`, etc.).
 - The **lock plus a shared cache directory** gives fleet-wide single-flight:
   the lock holder writes the cache on the shared volume, and every waiter reads
@@ -405,11 +429,12 @@ import { createUIMessageStream } from 'ai';
 
 import {
   AdapterIndexer,
+  FileIndexLock,
   TEXT2SQL_INDEX_PROGRESS_CHUNK,
   type Text2SqlIndexProgressEvent,
 } from '@deepagents/text2sql';
 
-const indexer = new AdapterIndexer({ adapters });
+const indexer = new AdapterIndexer({ adapters, lock: new FileIndexLock() });
 
 await context.continue(user('Show me top 10 customers'));
 

package/dist/bin/sql.js

@@ -68,6 +68,51 @@ var FileIndexCache = class {
   }
 };
 
+// packages/text2sql/src/lib/index-lock.ts
+import { createHash as createHash2 } from "node:crypto";
+import { mkdir as mkdir2 } from "node:fs/promises";
+import { tmpdir as tmpdir2 } from "node:os";
+import * as path2 from "node:path";
+import { lock as acquireLock } from "proper-lockfile";
+var DEFAULT_STALE_MS = 3e4;
+var DEFAULT_RETRIES = {
+  retries: 10,
+  factor: 2,
+  minTimeout: 100,
+  maxTimeout: 2e3
+};
+var FileIndexLock = class {
+  #dir;
+  #namespace;
+  #stale;
+  #retries;
+  constructor(options = {}) {
+    this.#dir = options.dir ?? tmpdir2();
+    this.#namespace = options.namespace;
+    this.#stale = options.stale ?? DEFAULT_STALE_MS;
+    this.#retries = options.retries ?? DEFAULT_RETRIES;
+  }
+  async run(key, fn) {
+    await mkdir2(this.#dir, { recursive: true });
+    const release = await acquireLock(this.#lockPathFor(key), {
+      realpath: false,
+      stale: this.#stale,
+      retries: this.#retries
+    });
+    try {
+      return await fn();
+    } finally {
+      await release().catch(() => {
+      });
+    }
+  }
+  #lockPathFor(key) {
+    const watermark = this.#namespace ? `${this.#namespace}-${key}` : key;
+    const hash = createHash2("md5").update(watermark).digest("hex");
+    return path2.join(this.#dir, `text2sql-lock-${hash}`);
+  }
+};
+
 // packages/text2sql/src/lib/sql.ts
 import "@deepagents/agent";
 
@@ -76,6 +121,8 @@ import { fragment } from "@deepagents/context";
 
 // packages/text2sql/src/lib/adapter-name.ts
 var ADAPTER_NAME_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/;
+var DATABASE_NAME_FRAGMENT = "database";
+var RESERVED_ADAPTER_NAMES = /* @__PURE__ */ new Set([DATABASE_NAME_FRAGMENT]);
 function isValidAdapterName(name) {
   return name != null && ADAPTER_NAME_PATTERN.test(name);
 }
@@ -86,6 +133,9 @@ function validateAdapterNames(names) {
         `Invalid adapter name "${name}": must match ${ADAPTER_NAME_PATTERN}`
       );
     }
+    if (RESERVED_ADAPTER_NAMES.has(name)) {
+      throw new Error(`Adapter name "${name}" is reserved`);
+    }
   }
 }
 
@@ -197,6 +247,9 @@ function timestampProgressHandler(onProgress) {
     });
   };
 }
+function databaseNameFragment(name) {
+  return { name: DATABASE_NAME_FRAGMENT, data: name };
+}
 var AdapterIndexer = class {
   #adapters;
   #cache;
@@ -223,7 +276,7 @@ var AdapterIndexer = class {
     const settled = await Promise.allSettled(
       entries.map(async ([name, adapter]) => {
         const schema = await this.#indexAdapter(name, adapter, progress);
-        return fragment(name, ...schema);
+        return fragment(name, databaseNameFragment(name), ...schema);
       })
     );
     const failed = settled.find((result) => result.status === "rejected");
@@ -310,7 +363,7 @@ var AdapterIndexer = class {
         });
         return cached;
       }
-      const fragments = this.#lock ? await this.#lock.run(key, populate) : await populate();
+      const fragments = await this.#lock.run(key, populate);
       if (servedFromCache) emitCacheHit();
       progress({
         type: "adapter:end",
@@ -926,14 +979,25 @@ var Text2Sql = class {
 };
 
 // packages/text2sql/src/bin/command.ts
-import * as path2 from "node:path";
+import * as path3 from "node:path";
+
+// packages/text2sql/src/lib/resolve-adapter.ts
+function resolveAdapter(text2Sql, db) {
+  if (text2Sql.hasAdapter(db)) return db;
+  const names = text2Sql.adapterNames();
+  if (names.length === 1) return names[0];
+  const available = names.join(", ") || "(none configured)";
+  throw new Error(`unknown database "${db}". Available: ${available}`);
+}
+
+// packages/text2sql/src/bin/command.ts
 var OUT_DIR_OPTION = {
   flag: "--out-dir <path>",
   description: "Output directory (default: $TEXT2SQL_OUT_DIR or ./sql)"
 };
 function resolveOutputDir(ctx, options) {
   const explicit = typeof options.outDir === "string" ? options.outDir : void 0;
-  return path2.resolve(ctx.cwd, explicit ?? ctx.env.TEXT2SQL_OUT_DIR ?? "./sql");
+  return path3.resolve(ctx.cwd, explicit ?? ctx.env.TEXT2SQL_OUT_DIR ?? "./sql");
 }
 var CommandError = class extends Error {
   command;
@@ -958,24 +1022,25 @@ var SqlCommand = class {
   fail(message, exitCode = 1) {
     throw new CommandError(this.name, message, exitCode);
   }
-  requireAdapterName(text2Sql, db) {
-    const available = text2Sql.adapterNames().join(", ") || "(none configured)";
+  resolveAdapterName(text2Sql, db) {
     if (!db) {
+      const available = text2Sql.adapterNames().join(", ") || "(none configured)";
       this.fail(
         `missing database name. Usage: sql ${this.name} ${this.usage}. Available: ${available}`
       );
     }
-    if (!text2Sql.hasAdapter(db)) {
-      this.fail(`unknown database "${db}". Available: ${available}`);
+    try {
+      return resolveAdapter(text2Sql, db);
+    } catch (error) {
+      this.fail(errorMessage2(error));
     }
-    return db;
   }
 };
 var SqlQueryCommand = class extends SqlCommand {
   args = "<db> <...sql>";
   async execute(ctx, args, options) {
     const [db, sqlParts] = args;
-    const name = this.requireAdapterName(ctx.text2Sql, db);
+    const name = this.resolveAdapterName(ctx.text2Sql, db);
     const sql = (sqlParts ?? []).join(" ").trim();
     if (!sql) this.fail("no query provided");
     try {
@@ -991,12 +1056,12 @@ var SqlQueryCommand = class extends SqlCommand {
 // packages/text2sql/src/bin/commands/index.ts
 import { constants, createWriteStream } from "node:fs";
 import {
-  mkdir as mkdir2,
+  mkdir as mkdir3,
   open,
   stat,
   writeFile as writeFile2
 } from "node:fs/promises";
-import * as path3 from "node:path";
+import * as path4 from "node:path";
 import { finished } from "node:stream/promises";
 import { v7 } from "uuid";
 
@@ -1040,13 +1105,13 @@ var IndexCommand = class extends SqlCommand {
     const verbose = this.resolveVerbose(options.verbose);
     const outputDir = resolveOutputDir(ctx, options);
     const id = v7();
-    const fragmentsPath = path3.join(outputDir, `index-${id}.json`);
+    const fragmentsPath = path4.join(outputDir, `index-${id}.json`);
     const explicitEventsPath = resolveExplicitEventsPath(ctx);
-    const eventsPath = explicitEventsPath ?? path3.join(outputDir, `index-${id}.events.ndjson`);
+    const eventsPath = explicitEventsPath ?? path4.join(outputDir, `index-${id}.events.ndjson`);
     try {
-      await mkdir2(outputDir, { recursive: true });
+      await mkdir3(outputDir, { recursive: true });
       if (explicitEventsPath) {
-        await mkdir2(path3.dirname(explicitEventsPath), { recursive: true });
+        await mkdir3(path4.dirname(explicitEventsPath), { recursive: true });
       }
       const eventsStream = await openEventsWriter(
         eventsPath,
@@ -1062,8 +1127,7 @@ var IndexCommand = class extends SqlCommand {
         const manifest = {
           fragmentsPath,
           eventsPath,
-          adapters: names,
-          fragments: countSchemaFragments(fragments)
+          adapters: names
         };
         ctx.stdout.write(JSON.stringify(manifest, null, 2) + "\n");
         return 0;
@@ -1107,7 +1171,7 @@ function createProgressHandler(eventsStream, verbose, stderr) {
 }
 function resolveExplicitEventsPath(ctx) {
   const explicit = ctx.env.TEXT2SQL_INDEX_EVENTS_PATH?.trim();
-  return explicit ? path3.resolve(ctx.cwd, explicit) : null;
+  return explicit ? path4.resolve(ctx.cwd, explicit) : null;
 }
 async function openEventsWriter(eventsPath, explicitEventsPath, stderr) {
   const probe = explicitEventsPath ? await probeFifoWriter(explicitEventsPath, stderr) : null;
@@ -1164,21 +1228,13 @@ function waitForOpen(stream) {
     stream.once("error", onError);
   });
 }
-function countSchemaFragments(fragments) {
-  return fragments.reduce((count, adapterFragment) => {
-    if (Array.isArray(adapterFragment.data)) {
-      return count + adapterFragment.data.length;
-    }
-    return count + 1;
-  }, 0);
-}
 function dedupe(values) {
   return [...new Set(values)];
 }
 
 // packages/text2sql/src/bin/commands/run.ts
-import { mkdir as mkdir3, writeFile as writeFile3 } from "node:fs/promises";
-import * as path4 from "node:path";
+import { mkdir as mkdir4, writeFile as writeFile3 } from "node:fs/promises";
+import * as path5 from "node:path";
 import { v7 as v72 } from "uuid";
 var RunCommand = class extends SqlQueryCommand {
   name = "run";
@@ -1189,9 +1245,9 @@ var RunCommand = class extends SqlQueryCommand {
   async runQuery(ctx, name, sql, options) {
     const outputDir = resolveOutputDir(ctx, options);
     const { rows, columns } = await ctx.text2Sql.run(name, sql);
-    const outPath = path4.join(outputDir, `${v72()}.json`);
+    const outPath = path5.join(outputDir, `${v72()}.json`);
     try {
-      await mkdir3(outputDir, { recursive: true });
+      await mkdir4(outputDir, { recursive: true });
       await writeFile3(outPath, JSON.stringify(rows, null, 2));
     } catch (error) {
       this.fail(errorMessage2(error));
@@ -1336,12 +1392,19 @@ function resolveIndexCache() {
   }
   return new FileIndexCache({ dir, namespace });
 }
+function resolveIndexLock() {
+  return new FileIndexLock({
+    dir: process.env.TEXT2SQL_INDEX_LOCK_DIR ?? process.env.TEXT2SQL_INDEX_CACHE_DIR,
+    namespace: process.env.TEXT2SQL_INDEX_VERSION
+  });
+}
 async function runCommand(command, positional, options) {
   let ctx;
   try {
     const text2Sql = new Text2Sql({
       adapters: await loadAdapters(),
-      cache: resolveIndexCache()
+      cache: resolveIndexCache(),
+      lock: resolveIndexLock()
     });
     ctx = {
       text2Sql,