@deepagents/text2sql 1.0.0 → 2.1.0

package/README.md

@@ -26,7 +26,7 @@ npm install mysql2                   # MySQL / MariaDB
 npm install @google-cloud/bigquery   # BigQuery
 ```
 
-Requires Node.js LTS (20+).
+Requires Node.js LTS
 
 ## Quick Start
 
@@ -95,10 +95,9 @@ Text2SQL works with any model provider supported by the [Vercel AI SDK](https://
 
 ## Building a Conversational Agent
 
-`Text2Sql` ships only the SQL-aware primitives: `toSql`, `toPairs`,
-`AdapterIndexer`, and `instructions()`. You build the chat agent yourself by
-composing a `ContextEngine`, a sandbox, and `agent` + `chat` from
-`@deepagents/context`:
+`Text2Sql` owns SQL generation and schema indexing, but you own the chat loop.
+Build the agent by composing a `ContextEngine`, a sandbox, and `agent` +
+`chat` from `@deepagents/context`:
 
 ```typescript
 import {
@@ -123,6 +122,14 @@ const context = new ContextEngine({
 
 const sandbox = await createContainerTool({
   installers: [npm('@deepagents/text2sql', { ensureRuntime: true })],
+  volumes: [
+    {
+      type: 'bind',
+      hostPath: process.cwd(),
+      containerPath: '/workspace',
+      readOnly: true,
+    },
+  ],
   env: {
     TEXT2SQL_ADAPTERS: '/workspace/text2sql-adapters.ts',
   },
@@ -154,6 +161,10 @@ for await (const chunk of stream) {
 }
 ```
 
+The `/workspace/text2sql-adapters.ts` module must exist in the sandbox and
+default-export your adapter map. Mount your project into `/workspace` (as above)
+or upload/write that module before you call `sql index` or `chat()`.
+
 `instructions()` returns the SQL-flavored system fragments (policies, workflows,
 clarifications, style guides) — spread them into `context.set()` alongside the
 schema fragments returned by `sql index`. Add or replace them with your
@@ -162,11 +173,25 @@ own domain fragments as needed.
 ## Advanced: SQL CLI in Sandboxes
 
 `sql validate <db> "..."` and `sql run <db> "..."` are real commands from the
-`@deepagents/text2sql` package. `sql index [adapter ...]` writes schema
-fragments plus progress events for chat setup. Install the package inside the
-sandbox and set `TEXT2SQL_ADAPTERS` to a module whose default export is
-`Record<string, Adapter>`. Missing `sql` means the sandbox was not prepared
-correctly.
+`@deepagents/text2sql` package. `sql index` writes schema fragments plus
+progress events for chat setup, indexing all configured adapters by default
+(same as `--all`) unless adapter names are provided.
+
+Install the package inside the sandbox and set `TEXT2SQL_ADAPTERS` to a module
+whose default export is `Record<string, Adapter>`. Missing `sql` means the
+sandbox was not prepared correctly.
+
+`sql index` output details:
+
+- `stdout`: JSON manifest with `fragmentsPath`, `eventsPath`, adapters, and
+  fragment count.
+- `--verbose pretty` or `--verbose json`: mirrors progress events to `stderr`
+  while keeping `stdout` as the manifest.
+- `--out-dir <path>`: writes artifacts under that path (default:
+  `$TEXT2SQL_OUT_DIR` or `./sql`).
+
+Set `TEXT2SQL_INDEX_VERSION` to manage cache invalidation across runs. Cache
+keys are `index-<version>-<adapter>`, so bump the version when schema changes.
 
 Spread `createSqlCommandHooks({ adapters })` into `createBashTool()` or
 `createContainerTool()` for model-driven bash calls. The before hook preserves
@@ -339,6 +364,7 @@ Full documentation available at [januarylabs.github.io/deepagents](https://janua
 
 - [Getting Started](https://januarylabs.github.io/deepagents/docs/text2sql/getting-started)
 - [Generate SQL](https://januarylabs.github.io/deepagents/docs/text2sql/to-sql)
+- [Agent + SQLite Store Recipe](https://januarylabs.github.io/deepagents/docs/text2sql/recipes/agent-sqlite-store)
 - [Teach the System](https://januarylabs.github.io/deepagents/docs/text2sql/teach-the-system)
 - [Build Conversations](https://januarylabs.github.io/deepagents/docs/text2sql/build-conversations)
 - [Grounding](https://januarylabs.github.io/deepagents/docs/text2sql/grounding)

package/dist/bin/sql.js

@@ -4,6 +4,15 @@
 import { cac } from "cac";
 
 // packages/text2sql/src/bin/command.ts
+import * as path from "node:path";
+var OUT_DIR_OPTION = {
+  flag: "--out-dir <path>",
+  description: "Output directory (default: $TEXT2SQL_OUT_DIR or ./sql)"
+};
+function resolveOutputDir(ctx, options) {
+  const explicit = typeof options.outDir === "string" ? options.outDir : void 0;
+  return path.resolve(ctx.cwd, explicit ?? ctx.env.TEXT2SQL_OUT_DIR ?? "./sql");
+}
 var CommandError = class extends Error {
   command;
   exitCode;
@@ -13,15 +22,11 @@ var CommandError = class extends Error {
     this.command = command;
     this.exitCode = exitCode;
   }
-  toResult() {
-    return {
-      stdout: "",
-      stderr: `sql ${this.command}: ${this.message}
-`,
-      exitCode: this.exitCode
-    };
-  }
 };
+function renderCommandError(error, stderr) {
+  stderr.write(`sql ${error.command}: ${error.message}
+`);
+}
 function errorMessage(error) {
   return error instanceof Error ? error.message : String(error);
 }
@@ -73,9 +78,15 @@ var SqlQueryCommand = class extends SqlCommand {
 };
 
 // packages/text2sql/src/bin/commands/index.ts
-import { appendFileSync } from "node:fs";
-import { mkdir, writeFile as writeFile2 } from "node:fs/promises";
-import * as path2 from "node:path";
+import { constants, createWriteStream } from "node:fs";
+import {
+  mkdir,
+  open,
+  stat,
+  writeFile as writeFile2
+} from "node:fs/promises";
+import * as path3 from "node:path";
+import { finished } from "node:stream/promises";
 import { v7 } from "uuid";
 
 // packages/text2sql/src/lib/adapter-index.ts
@@ -186,12 +197,12 @@ import { createHash } from "node:crypto";
 import { existsSync } from "node:fs";
 import { readFile, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
-import path from "node:path";
+import path2 from "node:path";
 var FileCache = class {
   path;
   constructor(watermark, extension = ".txt") {
     const hash = createHash("md5").update(watermark).digest("hex");
-    this.path = path.join(tmpdir(), `text2sql-${hash}${extension}`);
+    this.path = path2.join(tmpdir(), `text2sql-${hash}${extension}`);
   }
   async get() {
     if (existsSync(this.path)) {
@@ -369,7 +380,7 @@ var AdapterIndexer = class {
       return configured;
     }
     if (this.#version) {
-      return `introspection-${this.#version}-${name}`;
+      return `index-${this.#version}-${name}`;
     }
     return void 0;
   }
@@ -391,44 +402,82 @@ function errorMessage2(error) {
   return error instanceof Error ? error.message : String(error);
 }
 
+// packages/text2sql/src/bin/commands/indexing-formatter.ts
+var REDUNDANT_MESSAGE_TYPES = /* @__PURE__ */ new Set([
+  "adapter:start",
+  "adapter:end"
+]);
+function formatPretty(event) {
+  const head = event.adapter ? `${event.type} ${event.adapter}` : event.type;
+  const parts = [`[${head}]`];
+  if (event.phase) parts.push(`phase=${event.phase}`);
+  if (event.table) parts.push(`table=${event.table}`);
+  if (typeof event.current === "number" && typeof event.total === "number" && !event.type.endsWith(":start")) {
+    parts.push(`${event.current}/${event.total}`);
+  }
+  if (event.cached) parts.push("cached");
+  if (!REDUNDANT_MESSAGE_TYPES.has(event.type)) parts.push(event.message);
+  return parts.join(" ");
+}
+
 // packages/text2sql/src/bin/commands/index.ts
 var IndexCommand = class extends SqlCommand {
   name = "index";
   description = "Index adapter schemas and write context artifacts";
   args = "[...adapters]";
-  usage = "[--all] [adapter ...]";
-  helpDisplay = "[--all] [adapter ...]";
+  usage = "[--all] [--verbose [pretty|json]] [adapter ...]";
+  helpDisplay = "[--all] [--verbose] [adapter ...]";
   options = [
-    { flag: "--all", description: "Index all adapters (default)" }
+    { flag: "--all", description: "Index all adapters (default)" },
+    {
+      flag: "-v, --verbose [format]",
+      description: "Mirror progress events to stderr (format: pretty (default) | json)"
+    },
+    OUT_DIR_OPTION
   ];
   async execute(ctx, args, options) {
     const adapterNames = args[0] ?? [];
     const requested = options.all ? [] : adapterNames;
     const names = this.resolveNames(ctx.adapters, requested);
-    const outputDir = path2.resolve(ctx.cwd, "./sql");
+    const verbose = this.resolveVerbose(options.verbose);
+    const outputDir = resolveOutputDir(ctx, options);
     const id = v7();
-    const fragmentsPath = path2.join(outputDir, `index-${id}.json`);
-    const eventsPath = path2.join(outputDir, `index-${id}.events.ndjson`);
+    const fragmentsPath = path3.join(outputDir, `index-${id}.json`);
+    const explicitEventsPath = resolveExplicitEventsPath(ctx);
+    const eventsPath = explicitEventsPath ?? path3.join(outputDir, `index-${id}.events.ndjson`);
     try {
       await mkdir(outputDir, { recursive: true });
-      await writeFile2(eventsPath, "");
-      const indexer = new AdapterIndexer({ adapters: ctx.adapters });
-      const fragments = await indexer.index({
-        adapterNames: names,
-        onProgress: (event) => appendProgressEvent(eventsPath, event)
-      });
-      await writeFile2(fragmentsPath, JSON.stringify(fragments, null, 2));
-      const manifest = {
-        fragmentsPath,
+      if (explicitEventsPath) {
+        await mkdir(path3.dirname(explicitEventsPath), { recursive: true });
+      }
+      const eventsStream = await openEventsWriter(
         eventsPath,
-        adapters: names,
-        fragments: countSchemaFragments(fragments)
-      };
-      return {
-        stdout: JSON.stringify(manifest, null, 2) + "\n",
-        stderr: "",
-        exitCode: 0
-      };
+        explicitEventsPath,
+        ctx.stderr
+      );
+      try {
+        const indexer = new AdapterIndexer({
+          adapters: ctx.adapters,
+          version: ctx.env.TEXT2SQL_INDEX_VERSION
+        });
+        const fragments = await indexer.index({
+          adapterNames: names,
+          onProgress: createProgressHandler(eventsStream, verbose, ctx.stderr)
+        });
+        await writeFile2(fragmentsPath, JSON.stringify(fragments, null, 2));
+        const manifest = {
+          fragmentsPath,
+          eventsPath,
+          adapters: names,
+          fragments: countSchemaFragments(fragments)
+        };
+        ctx.stdout.write(JSON.stringify(manifest, null, 2) + "\n");
+        return 0;
+      } finally {
+        eventsStream.end();
+        await finished(eventsStream).catch(() => {
+        });
+      }
     } catch (error) {
       this.fail(errorMessage(error));
     }
@@ -444,15 +493,82 @@ var IndexCommand = class extends SqlCommand {
     }
     return names;
   }
+  resolveVerbose(value) {
+    if (value === void 0 || value === false) return null;
+    if (value === true || value === "" || value === "pretty") return "pretty";
+    if (value === "json") return "json";
+    this.fail(
+      `invalid --verbose value "${String(value)}". Expected "pretty" or "json".`
+    );
+  }
 };
-function appendProgressEvent(eventsPath, event) {
-  appendFileSync(
-    eventsPath,
-    JSON.stringify({
-      ...event,
-      timestampMs: event.timestampMs ?? Date.now()
-    }) + "\n"
-  );
+function createProgressHandler(eventsStream, verbose, stderr) {
+  return (event) => {
+    const stamped = { ...event, timestampMs: event.timestampMs ?? Date.now() };
+    const json = JSON.stringify(stamped) + "\n";
+    eventsStream.write(json);
+    if (verbose === "pretty") stderr.write(formatPretty(stamped) + "\n");
+    else if (verbose === "json") stderr.write(json);
+  };
+}
+function resolveExplicitEventsPath(ctx) {
+  const explicit = ctx.env.TEXT2SQL_INDEX_EVENTS_PATH?.trim();
+  return explicit ? path3.resolve(ctx.cwd, explicit) : null;
+}
+async function openEventsWriter(eventsPath, explicitEventsPath, stderr) {
+  const probe = explicitEventsPath ? await probeFifoWriter(explicitEventsPath, stderr) : null;
+  const stream = createWriteStream(eventsPath, {
+    flags: "a",
+    encoding: "utf-8"
+  });
+  try {
+    await waitForOpen(stream);
+    return stream;
+  } catch (err) {
+    stream.destroy();
+    throw err;
+  } finally {
+    await probe?.close().catch(() => {
+    });
+  }
+}
+async function probeFifoWriter(eventsPath, stderr) {
+  let stats;
+  try {
+    stats = await stat(eventsPath);
+  } catch (err) {
+    if (err.code === "ENOENT") return null;
+    throw err;
+  }
+  if (!stats.isFIFO()) return null;
+  try {
+    return await open(eventsPath, constants.O_WRONLY | constants.O_NONBLOCK);
+  } catch (err) {
+    if (err.code === "ENXIO") {
+      stderr.write(`sql index: waiting for reader on FIFO ${eventsPath}...
+`);
+      return null;
+    }
+    throw err;
+  }
+}
+function waitForOpen(stream) {
+  return new Promise((resolve4, reject) => {
+    if (!stream.pending) {
+      resolve4();
+      return;
+    }
+    const onOpen = () => {
+      stream.off("error", onError);
+      resolve4();
+    };
+    const onError = (err) => {
+      stream.off("open", onOpen);
+      reject(err);
+    };
+    stream.once("open", onOpen);
+    stream.once("error", onError);
+  });
 }
 function countSchemaFragments(fragments) {
   return fragments.reduce((count, adapterFragment) => {
@@ -468,21 +584,16 @@ function dedupe(values) {
 
 // packages/text2sql/src/bin/commands/run.ts
 import { mkdir as mkdir2, writeFile as writeFile3 } from "node:fs/promises";
-import * as path3 from "node:path";
+import * as path4 from "node:path";
 import { v7 as v72 } from "uuid";
 var RunCommand = class extends SqlQueryCommand {
   name = "run";
   description = "Execute query against <db> and store results";
   usage = '<db> "SELECT ..." [--out-dir <path>]';
   helpDisplay = '<db> "SELECT ..."';
-  options = [
-    {
-      flag: "--out-dir <path>",
-      description: "Directory for `sql run` result files (default: $TEXT2SQL_OUT_DIR or ./sql)"
-    }
-  ];
+  options = [OUT_DIR_OPTION];
   async afterValidation(ctx, adapter, sql, options) {
-    const outputDir = this.resolveOutputDir(ctx, options);
+    const outputDir = resolveOutputDir(ctx, options);
     let rows;
     try {
       rows = await adapter.execute(sql);
@@ -492,7 +603,7 @@ var RunCommand = class extends SqlQueryCommand {
     if (!Array.isArray(rows)) {
       this.fail("adapter.execute must return an array of rows");
     }
-    const outPath = path3.join(outputDir, `${v72()}.json`);
+    const outPath = path4.join(outputDir, `${v72()}.json`);
     try {
       await mkdir2(outputDir, { recursive: true });
       await writeFile3(outPath, JSON.stringify(rows, null, 2));
@@ -500,22 +611,13 @@ var RunCommand = class extends SqlQueryCommand {
       this.fail(errorMessage(error));
     }
     const columns = rows.length > 0 ? Object.keys(rows[0]) : [];
-    return {
-      stdout: [
-        `results stored in ${outPath}`,
-        `columns: ${columns.join(", ") || "(none)"}`,
-        `rows: ${rows.length}`
-      ].join("\n") + "\n",
-      stderr: "",
-      exitCode: 0
-    };
-  }
-  resolveOutputDir(ctx, options) {
-    const explicit = typeof options.outDir === "string" ? options.outDir : void 0;
-    return path3.resolve(
-      ctx.cwd,
-      explicit ?? ctx.env.TEXT2SQL_OUT_DIR ?? "./sql"
-    );
+    ctx.stdout.write(`results stored in ${outPath}
+`);
+    ctx.stdout.write(`columns: ${columns.join(", ") || "(none)"}
+`);
+    ctx.stdout.write(`rows: ${rows.length}
+`);
+    return 0;
   }
 };
 
@@ -525,8 +627,9 @@ var ValidateCommand = class extends SqlQueryCommand {
   description = "Validate query syntax against <db>";
   usage = '<db> "SELECT ..."';
   helpDisplay = '<db> "SELECT ..."';
-  async afterValidation() {
-    return { stdout: "valid\n", stderr: "", exitCode: 0 };
+  async afterValidation(ctx) {
+    ctx.stdout.write("valid\n");
+    return 0;
   }
 };
 
@@ -603,7 +706,7 @@ cli.usage("<run|validate|index> ...").help(
 for (const command of commands) {
   registerCommand(cli, command);
 }
-process.exit(await runCli());
+process.exitCode = await runCli();
 async function runCli() {
   try {
     cli.parse(process.argv, { run: false });
@@ -645,7 +748,9 @@ async function runCommand(command, positional, options) {
     ctx = {
       adapters: await loadAdapters(),
       cwd: process.cwd(),
-      env: process.env
+      env: process.env,
+      stdout: process.stdout,
+      stderr: process.stderr
     };
   } catch (error) {
     process.stderr.write(`${errorMessage(error)}
@@ -653,12 +758,10 @@ async function runCommand(command, positional, options) {
     return 2;
   }
   try {
-    const result = await command.execute(ctx, positional, options);
-    writeCommandResult(result);
-    return result.exitCode;
+    return await command.execute(ctx, positional, options);
   } catch (error) {
     if (error instanceof CommandError) {
-      writeCommandResult(error.toResult());
+      renderCommandError(error, process.stderr);
       return error.exitCode;
     }
     throw error;
@@ -673,13 +776,6 @@ function rewriteCommandsHelp(body) {
     );
   }, body);
 }
-function writeCommandResult(result) {
-  if (result.stdout) process.stdout.write(result.stdout);
-  if (result.stderr) {
-    const out = result.stderr.endsWith("\n") ? result.stderr : result.stderr + "\n";
-    process.stderr.write(out);
-  }
-}
 function writeCliError(error) {
   if (error instanceof Error && error.name === "CACError") {
     process.stderr.write(`sql: ${error.message}
@@ -688,7 +784,7 @@ function writeCliError(error) {
     return 2;
   }
   if (error instanceof CommandError) {
-    writeCommandResult(error.toResult());
+    renderCommandError(error, process.stderr);
     return error.exitCode;
   }
   process.stderr.write(

package/dist/index.js

@@ -1442,7 +1442,7 @@ var AdapterIndexer = class {
       return configured;
     }
     if (this.#version) {
-      return `introspection-${this.#version}-${name}`;
+      return `index-${this.#version}-${name}`;
     }
     return void 0;
   }