@deepagents/text2sql 1.0.0 → 2.2.0

package/dist/bin/sql.js

@@ -3,80 +3,8 @@
 // packages/text2sql/src/bin/sql.ts
 import { cac } from "cac";
 
-// packages/text2sql/src/bin/command.ts
-var CommandError = class extends Error {
-  command;
-  exitCode;
-  constructor(command, message, exitCode = 1) {
-    super(message);
-    this.name = "CommandError";
-    this.command = command;
-    this.exitCode = exitCode;
-  }
-  toResult() {
-    return {
-      stdout: "",
-      stderr: `sql ${this.command}: ${this.message}
-`,
-      exitCode: this.exitCode
-    };
-  }
-};
-function errorMessage(error) {
-  return error instanceof Error ? error.message : String(error);
-}
-var SqlCommand = class {
-  options = [];
-  helpDisplay;
-  fail(message, exitCode = 1) {
-    throw new CommandError(this.name, message, exitCode);
-  }
-  requireAdapter(adapters, db) {
-    const available = Object.keys(adapters).join(", ") || "(none configured)";
-    if (!db) {
-      this.fail(
-        `missing database name. Usage: sql ${this.name} ${this.usage}. Available: ${available}`
-      );
-    }
-    const adapter = adapters[db];
-    if (!adapter) {
-      this.fail(`unknown database "${db}". Available: ${available}`);
-    }
-    return adapter;
-  }
-};
-var SqlQueryCommand = class extends SqlCommand {
-  args = "<db> <...sql>";
-  async execute(ctx, args, options) {
-    const [db, sqlParts] = args;
-    const adapter = this.requireAdapter(ctx.adapters, db);
-    const sql = (sqlParts ?? []).join(" ").trim();
-    if (!sql) this.fail("no query provided");
-    const formatted = await this.formatAndValidate(adapter, sql);
-    return this.afterValidation(ctx, adapter, formatted, options);
-  }
-  async formatAndValidate(adapter, sql) {
-    let formatted;
-    try {
-      formatted = adapter.format(sql);
-    } catch (error) {
-      this.fail(errorMessage(error));
-    }
-    try {
-      const syntaxError = await adapter.validate(formatted);
-      if (syntaxError) this.fail(syntaxError);
-    } catch (error) {
-      this.fail(errorMessage(error));
-    }
-    return formatted;
-  }
-};
-
-// packages/text2sql/src/bin/commands/index.ts
-import { appendFileSync } from "node:fs";
-import { mkdir, writeFile as writeFile2 } from "node:fs/promises";
-import * as path2 from "node:path";
-import { v7 } from "uuid";
+// packages/text2sql/src/lib/sql.ts
+import "@deepagents/agent";
 
 // packages/text2sql/src/lib/adapter-index.ts
 import { fragment } from "@deepagents/context";
@@ -180,6 +108,17 @@ function createGroundingContext(options = {}) {
 import nodeSqlParser from "node-sql-parser";
 var { Parser } = nodeSqlParser;
 var parser = new Parser();
+function buildScopeParseErrorPayload(sql, dialect, error) {
+  const parserError = error instanceof Error ? error.message : String(error ?? "Unknown error");
+  return {
+    error: `SQL scope analysis failed before validation/execution: ${parserError}`,
+    error_type: "SQL_SCOPE_PARSE_ERROR",
+    suggestion: "Rewrite the query into simpler SQL that can be analyzed safely, or extend parser coverage for this dialect feature.",
+    sql_attempted: sql,
+    parser_dialect: dialect,
+    parser_error: parserError
+  };
+}
 
 // packages/text2sql/src/lib/file-cache.ts
 import { createHash } from "node:crypto";
@@ -340,7 +279,7 @@ var AdapterIndexer = class {
       });
       return fragments;
     } catch (error) {
-      const reason = errorMessage2(error);
+      const reason = errorMessage(error);
       progress({
         type: "adapter:error",
         adapter: name,
@@ -369,7 +308,7 @@ var AdapterIndexer = class {
       return configured;
     }
     if (this.#version) {
-      return `introspection-${this.#version}-${name}`;
+      return `index-${this.#version}-${name}`;
     }
     return void 0;
   }
@@ -387,72 +326,811 @@ function adapterProgressEvent(adapter, progress) {
     timestampMs: progress.timestampMs
   };
 }
+function errorMessage(error) {
+  return error instanceof Error ? error.message : String(error);
+}
+
+// packages/text2sql/src/lib/agents/sql.agent.ts
+import { groq } from "@ai-sdk/groq";
+import {
+  APICallError,
+  JSONParseError,
+  NoContentGeneratedError,
+  NoObjectGeneratedError,
+  NoOutputGeneratedError,
+  TypeValidationError,
+  defaultSettingsMiddleware,
+  wrapLanguageModel
+} from "ai";
+import dedent from "dedent";
+import pRetry from "p-retry";
+import z from "zod";
+import "@deepagents/agent";
+import {
+  ContextEngine,
+  InMemoryContextStore,
+  example,
+  fragment as fragment2,
+  guardrail,
+  hint,
+  persona,
+  policy,
+  structuredOutput,
+  user,
+  workflow
+} from "@deepagents/context";
+var RETRY_TEMPERATURES = [0, 0.2, 0.3];
+var SQL_AGENT_ROLE = "Expert SQL query generator.";
+var SQL_AGENT_OBJECTIVE = "Generate precise SQL grounded in provided schema.";
+var FORMATTER_SCOPE_DIALECTS = /* @__PURE__ */ new Map([
+  ["sqlite", "sqlite"],
+  ["postgresql", "postgresql"],
+  ["bigquery", "bigquery"],
+  ["transactsql", "transactsql"],
+  ["mysql", "mysql"]
+]);
+var SQL_AGENT_POLICIES = [
+  fragment2(
+    "schema_mapping",
+    policy({
+      rule: "Translate natural language into precise SQL grounded in available schema entities."
+    }),
+    hint("Preserve schema spelling exactly, including typos in column names.")
+  ),
+  fragment2(
+    "projection_minimality",
+    policy({
+      rule: "Return only columns requested by the question; do not add helper columns unless explicitly requested."
+    }),
+    policy({
+      rule: 'For requests of the form "X sorted/ordered by Y", project X only unless Y is explicitly requested as an output field.'
+    }),
+    policy({
+      rule: "Prefer selecting schema columns directly without derived expressions when direct selection answers the request."
+    }),
+    hint(
+      "Do not include ORDER BY, GROUP BY, or JOIN helper columns in SELECT output unless the question explicitly asks for them."
+    ),
+    policy({
+      rule: "Use DISTINCT only when uniqueness is explicitly requested (for example distinct/unique/different/no duplicates)."
+    }),
+    hint(
+      'Do not infer DISTINCT from generic wording such as "some", plural nouns, or entity-set phrasing; for transactional/attendance-style tables, default to raw rows unless uniqueness is explicitly requested.'
+    )
+  ),
+  fragment2(
+    "date_transform_safety",
+    policy({
+      rule: "Do not assume VARCHAR/TEXT values are parseable dates. Avoid date extraction functions on text columns by default."
+    }),
+    policy({
+      rule: "Use date-part extraction only when both conditions hold: the question explicitly asks for transformation and schema values require transformation to produce that unit."
+    }),
+    hint(
+      "Do not apply SUBSTR, STRFTIME, DATE_PART, YEAR, or similar extraction functions unless the question explicitly asks for transformation and schema values require it."
+    ),
+    hint(
+      "If a column already represents the requested concept (for example a stored year-like value), use the column as-is."
+    )
+  ),
+  fragment2(
+    "sql_minimality",
+    guardrail({
+      rule: "Never hallucinate tables or columns.",
+      reason: "Schema fidelity is required.",
+      action: "Use only available schema entities."
+    }),
+    guardrail({
+      rule: "Avoid unnecessary transformations and derived projections.",
+      reason: "Extra transformations frequently change semantics and reduce correctness.",
+      action: "Do not add date parsing, substring extraction, or derived columns unless explicitly required by the question or schema."
+    })
+  ),
+  fragment2(
+    "preflight_checklist",
+    workflow({
+      task: "Final SQL preflight before returning output",
+      steps: [
+        "Verify selected columns match the question and remove unrequested helper projections.",
+        "If aggregate values are used only for ranking/filtering, keep them out of SELECT unless explicitly requested.",
+        "Prefer raw schema columns over derived expressions when raw columns already satisfy the request.",
+        "If a candidate query uses STRFTIME, SUBSTR, DATE_PART, YEAR, or similar extraction on text-like columns, remove that transformation unless explicitly required by the question.",
+        "Return only schema-grounded SQL using existing tables and columns."
+      ]
+    })
+  ),
+  fragment2(
+    "set_semantics",
+    policy({
+      rule: "For questions asking where both condition A and condition B hold over an attribute, compute the intersection of qualifying sets for that attribute."
+    }),
+    policy({
+      rule: "Do not force the same entity instance to satisfy both conditions unless the question explicitly requests the same person/row/entity."
+    }),
+    hint(
+      "Prefer INTERSECT (or logically equivalent set-based shape) over requiring the same physical row/entity to satisfy both conditions unless explicitly requested."
+    ),
+    hint(
+      "When two conditions describe different row groups whose shared attribute is requested, build each group separately and intersect the attribute values."
+    ),
+    hint(
+      "Do not collapse cross-group conditions into a single-row AND predicate when the intent is shared values across groups."
+    ),
+    policy({
+      rule: "If two predicates on the same field cannot both be true for one row, do not combine them with AND; use set operations across separate filtered subsets when shared values are requested."
+    })
+  ),
+  fragment2(
+    "predicate_column_alignment",
+    policy({
+      rule: "Match literal values to semantically compatible columns. Do not compare descriptive names to identifier columns."
+    }),
+    hint(
+      "When a filter value is a descriptive label (for example a department name), join through the lookup table and filter on its name/title column, not on *_id columns."
+    ),
+    hint(
+      "When relation roles are explicit in wording (for example host/home/source/destination), prefer foreign keys with matching role qualifiers over generic similarly named columns."
+    ),
+    policy({
+      rule: "When multiple foreign-key candidates exist, select the column whose qualifier best matches the relationship described in the question."
+    }),
+    policy({
+      rule: "For hosting/held semantics, prefer host_* relationship columns when available over generic *_id alternatives."
+    }),
+    hint(
+      'Interpret wording like "held/hosted a competition or event" as a hosting relationship and map to host_* foreign keys when present.'
+    ),
+    policy({
+      rule: "Do not compare descriptive labels or names to *_id columns; join to the table containing the descriptive field and filter there."
+    }),
+    policy({
+      rule: "Keep numeric identifiers unquoted when used as numeric equality filters unless schema indicates text identifiers."
+    }),
+    policy({
+      rule: "When filtering by a descriptive label value and a related table exposes a corresponding *_name or title column, join to that table and filter on the descriptive column."
+    })
+  ),
+  fragment2(
+    "ordering_semantics",
+    policy({
+      rule: "Respect explicit sort direction terms. If direction is not specified, use ascending order unless a superlative intent (most/least/highest/lowest) implies direction."
+    }),
+    policy({
+      rule: "When ranking categories by frequency, use COUNT for ordering but keep output focused on requested category fields unless counts are explicitly requested."
+    }),
+    policy({
+      rule: "Do not use DESC unless descending direction is explicit or a superlative intent requires descending ranking."
+    }),
+    policy({
+      rule: 'For "most common/frequent <attribute>" requests, return the attribute value(s) only; use counts only for ordering/filtering unless the question explicitly asks to return counts.'
+    }),
+    hint(
+      'Use DESC with LIMIT 1 for "most/highest/largest"; use ASC with LIMIT 1 for "least/lowest/smallest".'
+    )
+  ),
+  fragment2(
+    "negative_membership_queries",
+    policy({
+      rule: "For requests asking entities that did not participate/host/appear in related records, prefer NOT IN or NOT EXISTS against the related foreign-key set."
+    }),
+    hint(
+      "Map role-bearing relationship columns carefully (for example host_* foreign keys for hosting relationships) instead of generic IDs when role wording is explicit."
+    ),
+    hint(
+      'For "never had/never exceeded" conditions over history tables, exclude entities via NOT IN/NOT EXISTS against the disqualifying entity-id set (often built with GROUP BY/HAVING MAX(...)).'
+    )
+  ),
+  fragment2(
+    "join_completeness",
+    policy({
+      rule: "Preserve entity-restricting joins implied by the question. Do not widen results by querying only a broader attribute table when a subset entity table is available."
+    }),
+    policy({
+      rule: "If an entity term in the question maps to a table, keep that table in query scope and join to attribute tables rather than dropping the entity table."
+    }),
+    hint(
+      "If the question targets a specific entity group, include that entity table and its join conditions even when selected columns come from a related table."
+    ),
+    hint(
+      "When the question names an entity type and a relation table links to that entity via *_id, include the entity table in scope instead of counting only relation rows."
+    ),
+    hint(
+      "Prefer INNER JOIN by default; use LEFT JOIN only when the question explicitly requests including unmatched rows or zero-related entities."
+    )
+  ),
+  fragment2(
+    "aggregation_exactness",
+    policy({
+      rule: "Preserve requested aggregation semantics exactly: use COUNT(*) by default for total rows, use COUNT(DISTINCT ...) only when uniqueness is explicitly requested, and group by stable entity keys when computing per-entity aggregates."
+    }),
+    policy({
+      rule: "For questions asking which entity has lowest/highest average of a metric, compute AVG(metric) per entity (GROUP BY entity) and rank those aggregates."
+    }),
+    hint(
+      'For "how many <entities>" questions over relation records, default to COUNT(*) on qualifying rows unless explicit uniqueness language is present.'
+    )
+  ),
+  fragment2(
+    "query_shape_examples",
+    example({
+      question: "List categories ordered by how many records belong to each category.",
+      answer: "SELECT category FROM records GROUP BY category ORDER BY COUNT(*)"
+    }),
+    example({
+      question: "Show labels shared by rows with metric > 100 and rows with metric < 10.",
+      answer: "SELECT label FROM records WHERE metric > 100 INTERSECT SELECT label FROM records WHERE metric < 10"
+    }),
+    example({
+      question: "List locations that have not hosted any event.",
+      answer: "SELECT location_name FROM locations WHERE location_id NOT IN (SELECT host_location_id FROM events)"
+    }),
+    example({
+      question: "List the most common category across records.",
+      answer: "SELECT category FROM records GROUP BY category ORDER BY COUNT(*) DESC LIMIT 1"
+    })
+  )
+];
+function extractSql(output) {
+  const match = output.match(/```sql\n?([\s\S]*?)```/);
+  return match ? match[1].trim() : output.trim();
+}
+function scopeDialectFor(adapter) {
+  const dialect = FORMATTER_SCOPE_DIALECTS.get(adapter.formatterLanguage);
+  if (!dialect) {
+    throw new TypeError(
+      `No scope dialect mapping for formatter language "${adapter.formatterLanguage}".`
+    );
+  }
+  return dialect;
+}
+function fencedMarkdownParseError(sql, adapter) {
+  return new SQLValidationError(
+    JSON.stringify(
+      buildScopeParseErrorPayload(
+        sql,
+        scopeDialectFor(adapter),
+        new Error("SQL response was fenced markdown, not executable SQL.")
+      )
+    )
+  );
+}
+async function toSql(options) {
+  const { maxRetries = 3 } = options;
+  return withRetry(
+    async (attemptNumber, errors, attempts) => {
+      const context = new ContextEngine({
+        store: new InMemoryContextStore(),
+        chatId: `sql-gen-${crypto.randomUUID()}`,
+        userId: "system"
+      });
+      context.set(
+        persona({
+          name: "Freya",
+          role: SQL_AGENT_ROLE,
+          objective: SQL_AGENT_OBJECTIVE
+          // role: `You are a data science expert that provides well-reasoned and detailed responses.`,
+          // objective: `Your task is to understand the schema and generate a valid SQL query to answer the question. You first think about the reasoning process as an internal monologue and then provide the user with the answer.`,
+        }),
+        ...SQL_AGENT_POLICIES,
+        ...options.fragments
+      );
+      if (errors.length) {
+        const lastError = errors.at(-1);
+        context.set(
+          user(dedent`
+            Answer the following question with the SQL code. Use the piece of evidence and base your answer on the database schema.
+Given the question, the evidence and the database schema, return the SQL script that addresses the question.
+
+Question: ${options.input}
+`),
+          UnanswerableSQLError.isInstance(lastError) ? user(
+            `<retry_instruction>Your previous response marked the task as unanswerable. Re-evaluate using best-effort schema mapping. If the core intent is answerable with existing tables/columns, return SQL. Return error only when required core intent cannot be mapped without inventing schema elements.</retry_instruction>`
+          ) : user(
+            `<validation_error>Your previous SQL query had the following error: ${lastError?.message}. Please fix the query.</validation_error>`
+          )
+        );
+      } else {
+        context.set(
+          user(dedent`
+            Answer the following question with the SQL code. Use the piece of evidence and base your answer on the database schema.
+Given the question, the evidence and the database schema, return the SQL script that addresses the question.
+
+Question: ${options.input}
+`)
+        );
+      }
+      const temperature = RETRY_TEMPERATURES[attemptNumber - 1] ?? RETRY_TEMPERATURES[RETRY_TEMPERATURES.length - 1];
+      const baseModel = options.model ?? groq("openai/gpt-oss-20b");
+      const model = wrapLanguageModel({
+        model: baseModel,
+        middleware: defaultSettingsMiddleware({ settings: { temperature } })
+      });
+      const sqlOutput = structuredOutput({
+        model,
+        context,
+        schema: z.object({
+          result: z.union([
+            z.object({
+              sql: z.string().describe("The SQL query that answers the question"),
+              reasoning: z.string().describe("The reasoning steps taken to generate the SQL")
+            }),
+            z.object({
+              error: z.string().describe(
+                "Error message explaining why the question cannot be answered with the given schema"
+              )
+            })
+          ])
+        })
+      });
+      const { result: output } = await sqlOutput.generate();
+      const finalizeSql = async (rawSql) => {
+        const extractedSql = extractSql(rawSql);
+        const errorsOrUndefined = errors.length ? errors.map(formatErrorMessage) : void 0;
+        if (extractedSql === "") {
+          return {
+            attempts,
+            sql: "",
+            errors: errorsOrUndefined
+          };
+        }
+        const sql = options.adapter.format(extractedSql);
+        if (rawSql.trimStart().startsWith("```") && extractedSql === rawSql.trim()) {
+          throw fencedMarkdownParseError(sql, options.adapter);
+        }
+        const validationError = await options.adapter.validate(sql);
+        if (validationError) {
+          throw new SQLValidationError(validationError);
+        }
+        return {
+          attempts,
+          sql,
+          errors: errorsOrUndefined
+        };
+      };
+      if ("error" in output) {
+        context.set(
+          user(
+            "<best_effort_fallback>Do not return unanswerable. Produce the best valid SQL query that answers the core intent using only available schema entities.</best_effort_fallback>"
+          )
+        );
+        const forcedSqlOutput = structuredOutput({
+          model,
+          context,
+          schema: z.object({
+            sql: z.string().describe(
+              "Best-effort SQL query that answers the core intent using only available schema entities."
+            ),
+            reasoning: z.string().describe("Reasoning steps for best-effort schema mapping.")
+          })
+        });
+        try {
+          const forced = await forcedSqlOutput.generate();
+          return await finalizeSql(forced.sql);
+        } catch (error) {
+          if (SQLValidationError.isInstance(error) || APICallError.isInstance(error) || JSONParseError.isInstance(error) || TypeValidationError.isInstance(error) || NoObjectGeneratedError.isInstance(error) || NoOutputGeneratedError.isInstance(error) || NoContentGeneratedError.isInstance(error)) {
+            throw error;
+          }
+          throw new UnanswerableSQLError(output.error);
+        }
+      }
+      return await finalizeSql(output.sql);
+    },
+    { retries: maxRetries - 1 }
+  );
+}
+function formatErrorMessage(error) {
+  if (APICallError.isInstance(error)) {
+    if (error.message.startsWith("Failed to validate JSON")) {
+      return `Schema validation failed: ${error.message}`;
+    }
+    return error.message;
+  }
+  if (SQLValidationError.isInstance(error)) {
+    return `SQL Validation Error: ${error.message}`;
+  }
+  return error.message;
+}
+function isModelUnavailableError(error) {
+  if (!APICallError.isInstance(error)) {
+    return false;
+  }
+  const message = error.message.toLowerCase();
+  const responseBody = (error.responseBody ?? "").toLowerCase();
+  const is404ModelError = error.statusCode === 404 && (message.includes("model") || responseBody.includes("model_not_found"));
+  const errorCode = typeof error.data === "object" && error.data !== null && "error" in error.data && typeof error.data.error === "object" && error.data.error !== null && "code" in error.data.error && typeof error.data.error.code === "string" ? error.data.error.code.toLowerCase() : void 0;
+  return is404ModelError || errorCode === "model_not_found" || responseBody.includes('"code":"model_not_found"') || message.includes("model") && message.includes("does not exist or you do not have access to it");
+}
+async function withRetry(computation, options = { retries: 3 }) {
+  const errors = [];
+  let attempts = 0;
+  return pRetry(
+    (attemptNumber) => {
+      return computation(attemptNumber, errors, ++attempts);
+    },
+    {
+      retries: options.retries,
+      shouldRetry: (context) => {
+        if (UnanswerableSQLError.isInstance(context.error)) {
+          return false;
+        }
+        if (isModelUnavailableError(context.error)) {
+          return false;
+        }
+        if (SQLValidationError.isInstance(context.error)) {
+          return true;
+        }
+        console.log({
+          NoObjectGeneratedError: NoObjectGeneratedError.isInstance(
+            context.error
+          ),
+          NoOutputGeneratedError: NoOutputGeneratedError.isInstance(
+            context.error
+          ),
+          APICallError: APICallError.isInstance(context.error),
+          JSONParseError: JSONParseError.isInstance(context.error),
+          TypeValidationError: TypeValidationError.isInstance(context.error),
+          NoContentGeneratedError: NoContentGeneratedError.isInstance(
+            context.error
+          )
+        });
+        return APICallError.isInstance(context.error) || JSONParseError.isInstance(context.error) || TypeValidationError.isInstance(context.error) || NoObjectGeneratedError.isInstance(context.error) || NoOutputGeneratedError.isInstance(context.error) || NoContentGeneratedError.isInstance(context.error);
+      },
+      onFailedAttempt(context) {
+        errors.push(context.error);
+      }
+    }
+  );
+}
+
+// packages/text2sql/src/lib/sql.ts
+var text2SqlValidationMarker = Symbol("Text2SqlValidationError");
+var text2SqlUnknownAdapterMarker = Symbol("Text2SqlUnknownAdapterError");
+var Text2SqlValidationError = class _Text2SqlValidationError extends Error {
+  [text2SqlValidationMarker];
+  constructor(message) {
+    super(message);
+    this.name = "Text2SqlValidationError";
+    this[text2SqlValidationMarker] = true;
+  }
+  static isInstance(error) {
+    return error instanceof _Text2SqlValidationError && error[text2SqlValidationMarker] === true;
+  }
+};
+var Text2SqlUnknownAdapterError = class _Text2SqlUnknownAdapterError extends Error {
+  [text2SqlUnknownAdapterMarker];
+  adapter;
+  available;
+  constructor(adapter, available) {
+    const list = available.join(", ") || "(none configured)";
+    super(`Unknown adapter "${adapter}". Available: ${list}`);
+    this.name = "Text2SqlUnknownAdapterError";
+    this.adapter = adapter;
+    this.available = available;
+    this[text2SqlUnknownAdapterMarker] = true;
+  }
+  static isInstance(error) {
+    return error instanceof _Text2SqlUnknownAdapterError && error[text2SqlUnknownAdapterMarker] === true;
+  }
+};
+var Text2Sql = class {
+  #config;
+  #indexer;
+  constructor(config) {
+    const adapterNames = Object.keys(config.adapters);
+    if (adapterNames.length === 0) {
+      throw new Error("Text2Sql requires at least one adapter");
+    }
+    validateAdapterNames(adapterNames);
+    this.#config = config;
+    this.#indexer = new AdapterIndexer({
+      adapters: config.adapters,
+      version: config.version
+    });
+  }
+  #requireAdapter(name) {
+    const adapter = this.#config.adapters[name];
+    if (!adapter) {
+      throw new Text2SqlUnknownAdapterError(name, this.adapterNames());
+    }
+    return adapter;
+  }
+  #requireModel(op) {
+    const model = this.#config.model;
+    if (!model) {
+      throw new Error(`Text2Sql.${op}() requires a model in Text2SqlConfig`);
+    }
+    return model;
+  }
+  adapterNames() {
+    return Object.keys(this.#config.adapters);
+  }
+  hasAdapter(name) {
+    return name in this.#config.adapters;
+  }
+  async validate(name, sql) {
+    const adapter = this.#requireAdapter(name);
+    const result = this.#validateWith(adapter, sql);
+    return result;
+  }
+  async run(name, sql) {
+    const adapter = this.#requireAdapter(name);
+    const formatted = await this.#validateWith(adapter, sql);
+    const result = await adapter.execute(formatted);
+    if (!Array.isArray(result)) {
+      throw new Error("adapter.execute must return an array of rows");
+    }
+    const columns = result.length > 0 ? Object.keys(result[0]) : [];
+    return { rows: result, columns };
+  }
+  async #validateWith(adapter, sql) {
+    const formatted = adapter.format(sql);
+    const error = await adapter.validate(formatted);
+    if (error) throw new Text2SqlValidationError(error);
+    return formatted;
+  }
+  async index(options = {}) {
+    return this.#indexer.index({
+      adapterNames: options.names,
+      onProgress: options.onProgress
+    });
+  }
+  async toSql(input, adapterName) {
+    const adapter = this.#requireAdapter(adapterName);
+    const model = this.#requireModel("toSql");
+    const fragments = await this.#indexer.indexAdapter(adapterName);
+    const result = await toSql({
+      input,
+      adapter,
+      fragments,
+      model
+    });
+    return result.sql;
+  }
+  async toPairs(adapterName, factory) {
+    const adapter = this.#requireAdapter(adapterName);
+    const producer = factory(adapter);
+    return producer.toPairs();
+  }
+};
+
+// packages/text2sql/src/bin/command.ts
+import * as path2 from "node:path";
+var OUT_DIR_OPTION = {
+  flag: "--out-dir <path>",
+  description: "Output directory (default: $TEXT2SQL_OUT_DIR or ./sql)"
+};
+function resolveOutputDir(ctx, options) {
+  const explicit = typeof options.outDir === "string" ? options.outDir : void 0;
+  return path2.resolve(ctx.cwd, explicit ?? ctx.env.TEXT2SQL_OUT_DIR ?? "./sql");
+}
+var CommandError = class extends Error {
+  command;
+  exitCode;
+  constructor(command, message, exitCode = 1) {
+    super(message);
+    this.name = "CommandError";
+    this.command = command;
+    this.exitCode = exitCode;
+  }
+};
+function renderCommandError(error, stderr) {
+  stderr.write(`sql ${error.command}: ${error.message}
+`);
+}
 function errorMessage2(error) {
   return error instanceof Error ? error.message : String(error);
 }
+var SqlCommand = class {
+  options = [];
+  helpDisplay;
+  fail(message, exitCode = 1) {
+    throw new CommandError(this.name, message, exitCode);
+  }
+  requireAdapterName(text2Sql, db) {
+    const available = text2Sql.adapterNames().join(", ") || "(none configured)";
+    if (!db) {
+      this.fail(
+        `missing database name. Usage: sql ${this.name} ${this.usage}. Available: ${available}`
+      );
+    }
+    if (!text2Sql.hasAdapter(db)) {
+      this.fail(`unknown database "${db}". Available: ${available}`);
+    }
+    return db;
+  }
+};
+var SqlQueryCommand = class extends SqlCommand {
+  args = "<db> <...sql>";
+  async execute(ctx, args, options) {
+    const [db, sqlParts] = args;
+    const name = this.requireAdapterName(ctx.text2Sql, db);
+    const sql = (sqlParts ?? []).join(" ").trim();
+    if (!sql) this.fail("no query provided");
+    try {
+      return await this.runQuery(ctx, name, sql, options);
+    } catch (error) {
+      if (error instanceof CommandError) throw error;
+      if (Text2SqlValidationError.isInstance(error)) this.fail(error.message);
+      this.fail(errorMessage2(error));
+    }
+  }
+};
+
+// packages/text2sql/src/bin/commands/index.ts
+import { constants, createWriteStream } from "node:fs";
+import {
+  mkdir,
+  open,
+  stat,
+  writeFile as writeFile2
+} from "node:fs/promises";
+import * as path3 from "node:path";
+import { finished } from "node:stream/promises";
+import { v7 } from "uuid";
+
+// packages/text2sql/src/bin/commands/indexing-formatter.ts
+var REDUNDANT_MESSAGE_TYPES = /* @__PURE__ */ new Set([
+  "adapter:start",
+  "adapter:end"
+]);
+function formatPretty(event) {
+  const head = event.adapter ? `${event.type} ${event.adapter}` : event.type;
+  const parts = [`[${head}]`];
+  if (event.phase) parts.push(`phase=${event.phase}`);
+  if (event.table) parts.push(`table=${event.table}`);
+  if (typeof event.current === "number" && typeof event.total === "number" && !event.type.endsWith(":start")) {
+    parts.push(`${event.current}/${event.total}`);
+  }
+  if (event.cached) parts.push("cached");
+  if (!REDUNDANT_MESSAGE_TYPES.has(event.type)) parts.push(event.message);
+  return parts.join(" ");
+}
 
 // packages/text2sql/src/bin/commands/index.ts
 var IndexCommand = class extends SqlCommand {
   name = "index";
   description = "Index adapter schemas and write context artifacts";
   args = "[...adapters]";
-  usage = "[--all] [adapter ...]";
-  helpDisplay = "[--all] [adapter ...]";
+  usage = "[--all] [--verbose [pretty|json]] [adapter ...]";
+  helpDisplay = "[--all] [--verbose] [adapter ...]";
   options = [
-    { flag: "--all", description: "Index all adapters (default)" }
+    { flag: "--all", description: "Index all adapters (default)" },
+    {
+      flag: "-v, --verbose [format]",
+      description: "Mirror progress events to stderr (format: pretty (default) | json)"
+    },
+    OUT_DIR_OPTION
   ];
   async execute(ctx, args, options) {
     const adapterNames = args[0] ?? [];
     const requested = options.all ? [] : adapterNames;
-    const names = this.resolveNames(ctx.adapters, requested);
-    const outputDir = path2.resolve(ctx.cwd, "./sql");
+    const names = this.resolveNames(ctx.text2Sql, requested);
+    const verbose = this.resolveVerbose(options.verbose);
+    const outputDir = resolveOutputDir(ctx, options);
     const id = v7();
-    const fragmentsPath = path2.join(outputDir, `index-${id}.json`);
-    const eventsPath = path2.join(outputDir, `index-${id}.events.ndjson`);
+    const fragmentsPath = path3.join(outputDir, `index-${id}.json`);
+    const explicitEventsPath = resolveExplicitEventsPath(ctx);
+    const eventsPath = explicitEventsPath ?? path3.join(outputDir, `index-${id}.events.ndjson`);
     try {
       await mkdir(outputDir, { recursive: true });
-      await writeFile2(eventsPath, "");
-      const indexer = new AdapterIndexer({ adapters: ctx.adapters });
-      const fragments = await indexer.index({
-        adapterNames: names,
-        onProgress: (event) => appendProgressEvent(eventsPath, event)
-      });
-      await writeFile2(fragmentsPath, JSON.stringify(fragments, null, 2));
-      const manifest = {
-        fragmentsPath,
+      if (explicitEventsPath) {
+        await mkdir(path3.dirname(explicitEventsPath), { recursive: true });
+      }
+      const eventsStream = await openEventsWriter(
         eventsPath,
-        adapters: names,
-        fragments: countSchemaFragments(fragments)
-      };
-      return {
-        stdout: JSON.stringify(manifest, null, 2) + "\n",
-        stderr: "",
-        exitCode: 0
-      };
+        explicitEventsPath,
+        ctx.stderr
+      );
+      try {
+        const fragments = await ctx.text2Sql.index({
+          names,
+          onProgress: createProgressHandler(eventsStream, verbose, ctx.stderr)
+        });
+        await writeFile2(fragmentsPath, JSON.stringify(fragments, null, 2));
+        const manifest = {
+          fragmentsPath,
+          eventsPath,
+          adapters: names,
+          fragments: countSchemaFragments(fragments)
+        };
+        ctx.stdout.write(JSON.stringify(manifest, null, 2) + "\n");
+        return 0;
+      } finally {
+        eventsStream.end();
+        await finished(eventsStream).catch(() => {
+        });
+      }
     } catch (error) {
-      this.fail(errorMessage(error));
+      this.fail(errorMessage2(error));
     }
   }
-  resolveNames(adapters, requested) {
-    const availableNames = Object.keys(adapters);
+  resolveNames(text2Sql, requested) {
+    const availableNames = text2Sql.adapterNames();
     const available = availableNames.join(", ") || "(none configured)";
     const names = requested.length === 0 ? availableNames : dedupe(requested);
     for (const name of names) {
-      if (!adapters[name]) {
+      if (!text2Sql.hasAdapter(name)) {
         this.fail(`unknown adapter "${name}". Available: ${available}`);
       }
     }
     return names;
   }
+  resolveVerbose(value) {
+    if (value === void 0 || value === false) return null;
+    if (value === true || value === "" || value === "pretty") return "pretty";
+    if (value === "json") return "json";
+    this.fail(
+      `invalid --verbose value "${String(value)}". Expected "pretty" or "json".`
+    );
+  }
 };
-function appendProgressEvent(eventsPath, event) {
-  appendFileSync(
-    eventsPath,
-    JSON.stringify({
-      ...event,
-      timestampMs: event.timestampMs ?? Date.now()
-    }) + "\n"
-  );
+function createProgressHandler(eventsStream, verbose, stderr) {
+  return (event) => {
+    const stamped = { ...event, timestampMs: event.timestampMs ?? Date.now() };
+    const json = JSON.stringify(stamped) + "\n";
+    eventsStream.write(json);
+    if (verbose === "pretty") stderr.write(formatPretty(stamped) + "\n");
+    else if (verbose === "json") stderr.write(json);
+  };
+}
+function resolveExplicitEventsPath(ctx) {
+  const explicit = ctx.env.TEXT2SQL_INDEX_EVENTS_PATH?.trim();
+  return explicit ? path3.resolve(ctx.cwd, explicit) : null;
+}
+async function openEventsWriter(eventsPath, explicitEventsPath, stderr) {
+  const probe = explicitEventsPath ? await probeFifoWriter(explicitEventsPath, stderr) : null;
+  const stream = createWriteStream(eventsPath, {
+    flags: "a",
+    encoding: "utf-8"
+  });
+  try {
+    await waitForOpen(stream);
+    return stream;
+  } catch (err) {
+    stream.destroy();
+    throw err;
+  } finally {
+    await probe?.close().catch(() => {
+    });
+  }
+}
+async function probeFifoWriter(eventsPath, stderr) {
+  let stats;
+  try {
+    stats = await stat(eventsPath);
+  } catch (err) {
+    if (err.code === "ENOENT") return null;
+    throw err;
+  }
+  if (!stats.isFIFO()) return null;
+  try {
+    return await open(eventsPath, constants.O_WRONLY | constants.O_NONBLOCK);
+  } catch (err) {
+    if (err.code === "ENXIO") {
+      stderr.write(`sql index: waiting for reader on FIFO ${eventsPath}...
+`);
+      return null;
+    }
+    throw err;
+  }
+}
+function waitForOpen(stream) {
+  return new Promise((resolve4, reject) => {
+    if (!stream.pending) {
+      resolve4();
+      return;
+    }
+    const onOpen = () => {
+      stream.off("error", onError);
+      resolve4();
+    };
+    const onError = (err) => {
+      stream.off("open", onOpen);
+      reject(err);
+    };
+    stream.once("open", onOpen);
+    stream.once("error", onError);
+  });
 }
 function countSchemaFragments(fragments) {
   return fragments.reduce((count, adapterFragment) => {
@@ -468,54 +1146,31 @@ function dedupe(values) {
 
 // packages/text2sql/src/bin/commands/run.ts
 import { mkdir as mkdir2, writeFile as writeFile3 } from "node:fs/promises";
-import * as path3 from "node:path";
+import * as path4 from "node:path";
 import { v7 as v72 } from "uuid";
 var RunCommand = class extends SqlQueryCommand {
   name = "run";
   description = "Execute query against <db> and store results";
   usage = '<db> "SELECT ..." [--out-dir <path>]';
   helpDisplay = '<db> "SELECT ..."';
-  options = [
-    {
-      flag: "--out-dir <path>",
-      description: "Directory for `sql run` result files (default: $TEXT2SQL_OUT_DIR or ./sql)"
-    }
-  ];
-  async afterValidation(ctx, adapter, sql, options) {
-    const outputDir = this.resolveOutputDir(ctx, options);
-    let rows;
-    try {
-      rows = await adapter.execute(sql);
-    } catch (error) {
-      this.fail(errorMessage(error));
-    }
-    if (!Array.isArray(rows)) {
-      this.fail("adapter.execute must return an array of rows");
-    }
-    const outPath = path3.join(outputDir, `${v72()}.json`);
+  options = [OUT_DIR_OPTION];
+  async runQuery(ctx, name, sql, options) {
+    const outputDir = resolveOutputDir(ctx, options);
+    const { rows, columns } = await ctx.text2Sql.run(name, sql);
+    const outPath = path4.join(outputDir, `${v72()}.json`);
     try {
       await mkdir2(outputDir, { recursive: true });
       await writeFile3(outPath, JSON.stringify(rows, null, 2));
     } catch (error) {
-      this.fail(errorMessage(error));
+      this.fail(errorMessage2(error));
     }
-    const columns = rows.length > 0 ? Object.keys(rows[0]) : [];
-    return {
-      stdout: [
-        `results stored in ${outPath}`,
-        `columns: ${columns.join(", ") || "(none)"}`,
-        `rows: ${rows.length}`
-      ].join("\n") + "\n",
-      stderr: "",
-      exitCode: 0
-    };
-  }
-  resolveOutputDir(ctx, options) {
-    const explicit = typeof options.outDir === "string" ? options.outDir : void 0;
-    return path3.resolve(
-      ctx.cwd,
-      explicit ?? ctx.env.TEXT2SQL_OUT_DIR ?? "./sql"
-    );
+    ctx.stdout.write(`results stored in ${outPath}
+`);
+    ctx.stdout.write(`columns: ${columns.join(", ") || "(none)"}
+`);
+    ctx.stdout.write(`rows: ${rows.length}
+`);
+    return 0;
   }
 };
 
@@ -525,8 +1180,10 @@ var ValidateCommand = class extends SqlQueryCommand {
   description = "Validate query syntax against <db>";
   usage = '<db> "SELECT ..."';
   helpDisplay = '<db> "SELECT ..."';
-  async afterValidation() {
-    return { stdout: "valid\n", stderr: "", exitCode: 0 };
+  async runQuery(ctx, name, sql) {
+    await ctx.text2Sql.validate(name, sql);
+    ctx.stdout.write("valid\n");
+    return 0;
   }
 };
 
@@ -553,7 +1210,7 @@ async function loadAdapters() {
     mod = await import(specifier);
   } catch (cause) {
     throw new Error(
-      `TEXT2SQL_ADAPTERS=${target}: failed to import module - ${errorMessage(cause)}`
+      `TEXT2SQL_ADAPTERS=${target}: failed to import module - ${errorMessage2(cause)}`
     );
   }
   const exported = mod.default;
@@ -578,7 +1235,7 @@ async function loadAdapters() {
   try {
     validateAdapterNames(entries.map(([name]) => name));
   } catch (cause) {
-    throw new Error(`TEXT2SQL_ADAPTERS=${target}: ${errorMessage(cause)}`);
+    throw new Error(`TEXT2SQL_ADAPTERS=${target}: ${errorMessage2(cause)}`);
   }
   return exported;
 }
@@ -603,7 +1260,7 @@ cli.usage("<run|validate|index> ...").help(
 for (const command of commands) {
   registerCommand(cli, command);
 }
-process.exit(await runCli());
+process.exitCode = await runCli();
 async function runCli() {
   try {
     cli.parse(process.argv, { run: false });
@@ -642,23 +1299,27 @@ function registerCommand(cli2, command) {
 async function runCommand(command, positional, options) {
   let ctx;
   try {
-    ctx = {
+    const text2Sql = new Text2Sql({
       adapters: await loadAdapters(),
+      version: process.env.TEXT2SQL_INDEX_VERSION
+    });
+    ctx = {
+      text2Sql,
       cwd: process.cwd(),
-      env: process.env
+      env: process.env,
+      stdout: process.stdout,
+      stderr: process.stderr
     };
   } catch (error) {
-    process.stderr.write(`${errorMessage(error)}
+    process.stderr.write(`${errorMessage2(error)}
 `);
     return 2;
   }
   try {
-    const result = await command.execute(ctx, positional, options);
-    writeCommandResult(result);
-    return result.exitCode;
+    return await command.execute(ctx, positional, options);
   } catch (error) {
     if (error instanceof CommandError) {
-      writeCommandResult(error.toResult());
+      renderCommandError(error, process.stderr);
       return error.exitCode;
     }
     throw error;
@@ -673,13 +1334,6 @@ function rewriteCommandsHelp(body) {
     );
   }, body);
 }
-function writeCommandResult(result) {
-  if (result.stdout) process.stdout.write(result.stdout);
-  if (result.stderr) {
-    const out = result.stderr.endsWith("\n") ? result.stderr : result.stderr + "\n";
-    process.stderr.write(out);
-  }
-}
 function writeCliError(error) {
   if (error instanceof Error && error.name === "CACError") {
     process.stderr.write(`sql: ${error.message}
@@ -688,7 +1342,7 @@ function writeCliError(error) {
     return 2;
   }
   if (error instanceof CommandError) {
-    writeCommandResult(error.toResult());
+    renderCommandError(error, process.stderr);
     return error.exitCode;
   }
   process.stderr.write(