@deepagents/text2sql 4.3.0 → 6.0.0

package/CLICKHOUSE-DEPLOYMENT.md

@@ -0,0 +1,181 @@
+# ClickHouse adapter
+
+`@deepagents/text2sql/clickhouse` provides a client-independent ClickHouse
+adapter. It uses ClickHouse itself to classify SQL and resolve physical
+relations before either the consumer validator or the candidate query can run.
+
+## Supported versions
+
+The real-server behavior suite is pinned to:
+
+- `25.8.28.1`, the minimum supported version and previous LTS line; and
+- `26.6.1.1193`, the current stable line when this adapter was implemented.
+
+ClickHouse identifies 25.8 as an LTS release in the official
+[25.8 release material](https://clickhouse.com/blog/clickhouse-release-25-08),
+and published the current [26.6 release](https://clickhouse.com/blog/clickhouse-release-26-06)
+on July 3, 2026. Re-evaluate and extend the pinned matrix before changing this
+range. The official [support policy](https://clickhouse.com/legal/support-services-policy)
+defines the rolling stable and LTS support windows.
+
+## Usage
+
+The package does not depend on a ClickHouse client. Supply one raw query
+callback and keep protocol-specific result handling outside the adapter:
+
+```ts
+import {
+  ClickHouse,
+  constraints,
+  indexes,
+  info,
+  rowCount,
+  tables,
+  views,
+} from '@deepagents/text2sql/clickhouse';
+
+const query = async (sql: string) => {
+  const result = await client.query({ query: sql, format: 'JSON' });
+  return result.json();
+};
+
+const database = new ClickHouse({
+  defaultDatabase: 'analytics',
+  execute: query,
+  // EXPLAIN QUERY TREE already performs server-side semantic validation.
+  // Keep this hook for any additional application validation.
+  validate: async () => undefined,
+  grounding: [
+    info(),
+    tables({ filter: ['analytics.users', 'analytics.orders'] }),
+    views(),
+    rowCount(),
+    constraints(),
+    indexes(),
+  ],
+});
+```
+
+The raw callback must:
+
+- return row objects as an array, `{ data: rows }`, or `{ rows }`;
+- throw when ClickHouse reports an exception, including HTTP responses that
+  carry an `exception` field with a successful HTTP status;
+- select JSON as a client/protocol option (or HTTP `default_format`), not by
+  appending `FORMAT JSON` to the SQL; and
+- never call `database.execute()`, which would recurse through policy analysis.
+
+`execute` is intentionally the same raw callback used for policy inspection,
+grounding queries, and the final accepted SELECT.
+
+## Required database role
+
+Create a dedicated user with one dedicated role. Do not add direct grants or
+additional default roles to this user.
+
+```sql
+CREATE ROLE deepagents_readonly;
+
+GRANT SELECT ON analytics.* TO deepagents_readonly;
+
+-- Required by SQL policy analysis to reject SQL, executable, and WASM UDFs.
+GRANT SELECT ON system.functions TO deepagents_readonly;
+
+-- Required only when indexes() grounding is configured.
+GRANT SELECT ON system.data_skipping_indices TO deepagents_readonly;
+
+ALTER ROLE deepagents_readonly SETTINGS
+    readonly = 1,
+    max_execution_time = 30,
+    max_memory_usage = 2000000000,
+    max_rows_to_read = 100000000,
+    max_bytes_to_read = 5000000000,
+    max_result_rows = 100000,
+    max_threads = 4;
+
+CREATE USER deepagents
+IDENTIFIED WITH sha256_password BY 'replace-with-a-secret';
+
+GRANT deepagents_readonly TO deepagents;
+ALTER USER deepagents DEFAULT ROLE deepagents_readonly;
+```
+
+Use `readonly = 1`. The adapter checks the effective value together with
+`currentDatabase()` on first use, caches that successful check, and fails closed
+unless the callback returns exactly one valid row. Keep ClickHouse permissions
+and resource settings in place: server-assisted analysis is an additional
+boundary, not a replacement for RBAC.
+
+ClickHouse's official
+[agentic analytics guidance](https://clickhouse.com/blog/how-to-set-up-clickhouse-for-agentic-analytics)
+recommends a SELECT-only role, `readonly = 1`, and workload limits. In
+ClickHouse Cloud, use a read-only service for this workload as well. A read-only
+service can still export data through table functions, so service isolation
+does not replace the adapter checks.
+
+Use ClickHouse row policies for tenant filtering. The adapter does not rewrite
+arbitrary SQL and `EXPLAIN` output is not a mutable AST.
+
+## Policy behavior
+
+Both `validate()` and `execute()` independently run this sequence before a
+candidate can reach the corresponding consumer callback:
+
+1. verify effective `readonly = 1` and capture the effective database;
+2. inspect `EXPLAIN AST` and require one SELECT root;
+3. reject `INTO OUTFILE`, every table function, dictionary/Join-engine lookup
+   functions, and every non-system UDF origin;
+4. inspect `EXPLAIN QUERY TREE run_passes = 0` to retain unused CTEs;
+5. inspect the analyzed query tree to validate and resolve physical names; and
+6. compare the union of relations with grounded entities using exact,
+   case-sensitive, database-qualified names.
+
+Unknown response columns, AST nodes, query-tree nodes, relation shapes,
+function origins, parse failures, multiple statements, and malformed SQL all
+fail closed. Views and Distributed tables are authorized by the relation name
+reported by ClickHouse and must be grounded explicitly.
+Dictionary access through `dict*` and Join-engine access through `joinGet*` are
+not supported because those functions do not expose their data source as a
+normal relation.
+
+The pinned allowlist includes ordinary SELECTs, CTEs, views, joins, subqueries,
+unions, `ARRAY JOIN`, `PREWHERE`, grouping, `HAVING`, `LIMIT BY`, windows, and
+`QUALIFY`. Unapproved structural features remain rejected. For example, the
+current real-server fixtures deliberately fail closed on column transformers
+such as `* EXCEPT` and `ORDER BY ... WITH FILL` query-tree nodes.
+
+ClickHouse documents the AST and query-tree modes in its
+[EXPLAIN reference](https://clickhouse.com/docs/sql-reference/statements/explain).
+Because their textual diagnostics are not a stable parser API, extend an
+allowlist only after capturing output on both pinned versions and adding a
+public real-server behavior test.
+
+## Grounding behavior
+
+The subpath exports `info`, `tables`, `views`, `rowCount`, `constraints`, and
+`indexes` grounding factories.
+
+- `tables()` and `views()` read access-filtered `system.tables` and
+  `system.columns` metadata.
+- `constraints()` reports ClickHouse nullability, default expressions, and
+  primary-key columns. ClickHouse primary keys organize data and do not imply
+  uniqueness.
+- `indexes()` reports primary-key and data-skipping indexes; it needs the
+  explicit `system.data_skipping_indices` grant shown above.
+- `rowCount()` executes `count()` against each grounded table. Configure it
+  only when exact counts are worth the scan and retain the resource limits.
+- ClickHouse does not expose conventional foreign-key relationships, so table
+  traversal does not synthesize them.
+
+Keep grounding limited to curated databases and tables. A broad SELECT grant
+also broadens what schema metadata and physical relations can be authorized.
+
+## Compatibility testing
+
+The ClickHouse integration file uses only the public package subpath and starts
+real Docker servers. With Docker available, it runs both pinned images by
+default. Set `CLICKHOUSE_TEST_IMAGE` to one pinned image for a focused run.
+
+Do not replace the policy tests with mocked EXPLAIN payloads. New ClickHouse
+versions must prove their actual AST/query-tree shapes and security behavior
+against a real server before being added to the supported matrix.

package/README.md

@@ -5,7 +5,7 @@ AI-powered natural language to SQL. Ask questions in plain English, get executab
 ## Features
 
 - **Natural Language to SQL** - Convert questions to validated, executable queries
-- **Multi-Database Support** - PostgreSQL, SQLite, SQL Server, MySQL/MariaDB, and BigQuery adapters
+- **Multi-Database Support** - PostgreSQL, SQLite, SQL Server, MySQL/MariaDB, BigQuery, and ClickHouse adapters
 - **Schema-Aware** - Automatic introspection of tables, relationships, indexes, and constraints
 - **Domain Knowledge** - Inject business terms, guardrails, and query patterns via fragments
 - **Conversational** - Multi-turn conversations with context persistence
@@ -24,6 +24,7 @@ npm install pg                       # PostgreSQL
 npm install mssql                    # SQL Server
 npm install mysql2                   # MySQL / MariaDB
 npm install @google-cloud/bigquery   # BigQuery
+npm install @clickhouse/client       # ClickHouse, or use your preferred client
 ```
 
 Requires Node.js LTS
@@ -184,9 +185,13 @@ model must pass to `sql validate <db>` and `sql run <db>`. Add or replace the
 instructions with your own domain fragments as needed.
 
 `sqlValidateReminder()` returns a context reminder fragment backed by
-`SQL_VALIDATE_REMINDER`. It adds a tool-output system reminder when the current
-assistant turn calls `sql run <db> "..."` without a prior `sql validate <db>
-"..."` for the same configured adapter name and exact SQL text.
+`SQL_VALIDATE_REMINDER`. After a successful `sql run <db> "..."`, it adds a
+model-only tool-output reminder before the next generation when no prior
+`sql validate <db> "..."` used the same configured adapter name and exact SQL
+text. The raw SQL command result remains unchanged. During streamed `chat()`
+turns, the reminder is persisted as a synthetic user message for replay and
+prompt caching; bare `createPrepareStep()` integrations and
+`agent.generate()` own persistence of their generated assistant history.
 
 ## Advanced: SQL CLI in Sandboxes
 
@@ -267,9 +272,29 @@ blocks raw database access so read-only and scope checks stay behind
 metadata from the host adapter map without putting that concern into the real
 CLI.
 
-Read-only enforcement accepts a single `SELECT`/`WITH` statement even when it
-starts with whitespace or SQL comments (`-- ...`, `/* ... */`). It still
-rejects comment-only input, multi-statement batches, and write operations.
+Read-only enforcement parses a single `SELECT`/`WITH` statement even when it
+starts with whitespace or SQL comments (`-- ...`, `/* ... */`). Before any
+adapter validator or executor runs, the dialect policy rejects write
+statements, multi-statement batches, parser failures, `SELECT INTO`, locking
+reads/hints, assignments, known state/file/extension/remote-access functions,
+BigQuery external queries, qualified BigQuery persistent routines, and
+ClickHouse table functions or unsafe UDF origins.
+
+Scope checks compare referenced base tables and views against the entities
+produced by `tables()` and `views()` groundings. Entity-free queries such as
+`SELECT 1` do not run grounding just to resolve an empty allowlist, and metadata
+groundings such as `indexes()`, `constraints()`, `rowCount()`,
+`columnStats()`, and `columnValues()` annotate schema context without expanding
+the validation allowlist.
+
+This parser policy is one safety layer, not a database permission boundary.
+Static analysis cannot prove arbitrary unqualified user-defined functions are
+side-effect free, and an otherwise valid `SELECT` can still consume excessive
+CPU, memory, bytes, locks, or time. Production connections must use a dedicated
+least-privilege identity with `SELECT` only on intended relations; disable or
+withhold extension, file, remote-query, and routine execution capabilities; and
+configure dialect-appropriate statement timeouts, byte/result limits, and
+compute/memory limits.
 
 ## Fragments
 
@@ -516,6 +541,7 @@ Full documentation available at [januarylabs.github.io/deepagents](https://janua
 - [SQL Server](https://januarylabs.github.io/deepagents/docs/text2sql/sqlserver)
 - [MySQL / MariaDB](https://januarylabs.github.io/deepagents/docs/text2sql/mysql)
 - [BigQuery](https://januarylabs.github.io/deepagents/docs/text2sql/bigquery)
+- [ClickHouse](https://januarylabs.github.io/deepagents/docs/text2sql/clickhouse)
 
 ## Repository
 

package/dist/bin/sql.js

@@ -143,10 +143,10 @@ function validateAdapterNames(names) {
 import { format as formatSql } from "sql-formatter";
 
 // packages/text2sql/src/lib/agents/exceptions.ts
-var sqlValidationMarker = Symbol("SQLValidationError");
-var unanswerableSqlMarker = Symbol("UnanswerableSQLError");
-var sqlScopeMarker = Symbol("SQLScopeError");
-var sqlReadOnlyMarker = Symbol("SQLReadOnlyError");
+var sqlValidationMarker = /* @__PURE__ */ Symbol("SQLValidationError");
+var unanswerableSqlMarker = /* @__PURE__ */ Symbol("UnanswerableSQLError");
+var sqlScopeMarker = /* @__PURE__ */ Symbol("SQLScopeError");
+var sqlReadOnlyMarker = /* @__PURE__ */ Symbol("SQLReadOnlyError");
 var SQLValidationError = class _SQLValidationError extends Error {
   [sqlValidationMarker];
   constructor(message) {
@@ -219,22 +219,6 @@ function createGroundingContext(options = {}) {
   };
 }
 
-// packages/text2sql/src/lib/adapters/runtime-scope.ts
-import nodeSqlParser from "node-sql-parser";
-var { Parser } = nodeSqlParser;
-var parser = new Parser();
-function buildScopeParseErrorPayload(sql, dialect, error) {
-  const parserError = error instanceof Error ? error.message : String(error ?? "Unknown error");
-  return {
-    error: `SQL scope analysis failed before validation/execution: ${parserError}`,
-    error_type: "SQL_SCOPE_PARSE_ERROR",
-    suggestion: "Rewrite the query into simpler SQL that can be analyzed safely, or extend parser coverage for this dialect feature.",
-    sql_attempted: sql,
-    parser_dialect: dialect,
-    parser_error: parserError
-  };
-}
-
 // packages/text2sql/src/lib/adapter-index.ts
 var noopProgressHandler = () => {
 };
@@ -443,16 +427,24 @@ import {
   user,
   workflow
 } from "@deepagents/context";
+
+// packages/text2sql/src/lib/sql-scope-error.ts
+function buildScopeParseErrorPayload(sql, parserDialect, error) {
+  const parserError = error instanceof Error ? error.message : String(error ?? "Unknown error");
+  return {
+    error: `SQL scope analysis failed before validation/execution: ${parserError}`,
+    error_type: "SQL_SCOPE_PARSE_ERROR",
+    suggestion: "Rewrite the query into simpler SQL that can be analyzed safely, or extend parser coverage for this dialect feature.",
+    sql_attempted: sql,
+    parser_dialect: parserDialect,
+    parser_error: parserError
+  };
+}
+
+// packages/text2sql/src/lib/agents/sql.agent.ts
 var RETRY_TEMPERATURES = [0, 0.2, 0.3];
 var SQL_AGENT_ROLE = "Expert SQL query generator.";
 var SQL_AGENT_OBJECTIVE = "Generate precise SQL grounded in provided schema.";
-var FORMATTER_SCOPE_DIALECTS = /* @__PURE__ */ new Map([
-  ["sqlite", "sqlite"],
-  ["postgresql", "postgresql"],
-  ["bigquery", "bigquery"],
-  ["transactsql", "transactsql"],
-  ["mysql", "mysql"]
-]);
 var SQL_AGENT_POLICIES = [
   fragment2(
     "schema_mapping",
@@ -658,26 +650,6 @@ function extractSql(output) {
   const match = output.match(/```sql\n?([\s\S]*?)```/);
   return match ? match[1].trim() : output.trim();
 }
-function scopeDialectFor(adapter) {
-  const dialect = FORMATTER_SCOPE_DIALECTS.get(adapter.formatterLanguage);
-  if (!dialect) {
-    throw new TypeError(
-      `No scope dialect mapping for formatter language "${adapter.formatterLanguage}".`
-    );
-  }
-  return dialect;
-}
-function fencedMarkdownParseError(sql, adapter) {
-  return new SQLValidationError(
-    JSON.stringify(
-      buildScopeParseErrorPayload(
-        sql,
-        scopeDialectFor(adapter),
-        new Error("SQL response was fenced markdown, not executable SQL.")
-      )
-    )
-  );
-}
 async function toSql(options) {
   const { maxRetries = 3 } = options;
   return withRetry(
@@ -759,7 +731,17 @@ Question: ${options.input}
         }
         const sql = options.adapter.format(extractedSql);
         if (rawSql.trimStart().startsWith("```") && extractedSql === rawSql.trim()) {
-          throw fencedMarkdownParseError(sql, options.adapter);
+          throw new SQLValidationError(
+            JSON.stringify(
+              buildScopeParseErrorPayload(
+                sql,
+                options.adapter.formatterLanguage,
+                new Error(
+                  "SQL response was fenced markdown, not executable SQL."
+                )
+              )
+            )
+          );
         }
         const validationError = await options.adapter.validate(sql);
         if (validationError) {
@@ -867,8 +849,8 @@ async function withRetry(computation, options = { retries: 3 }) {
 }
 
 // packages/text2sql/src/lib/sql.ts
-var text2SqlValidationMarker = Symbol("Text2SqlValidationError");
-var text2SqlUnknownAdapterMarker = Symbol("Text2SqlUnknownAdapterError");
+var text2SqlValidationMarker = /* @__PURE__ */ Symbol("Text2SqlValidationError");
+var text2SqlUnknownAdapterMarker = /* @__PURE__ */ Symbol("Text2SqlUnknownAdapterError");
 var Text2SqlValidationError = class _Text2SqlValidationError extends Error {
   [text2SqlValidationMarker];
   constructor(message) {
@@ -896,7 +878,7 @@ var Text2SqlUnknownAdapterError = class _Text2SqlUnknownAdapterError extends Err
     return error instanceof _Text2SqlUnknownAdapterError && error[text2SqlUnknownAdapterMarker] === true;
   }
 };
-var text2SqlUnknownDatabaseMarker = Symbol("Text2SqlUnknownDatabaseError");
+var text2SqlUnknownDatabaseMarker = /* @__PURE__ */ Symbol("Text2SqlUnknownDatabaseError");
 var Text2SqlUnknownDatabaseError = class _Text2SqlUnknownDatabaseError extends Error {
   [text2SqlUnknownDatabaseMarker];
   requested;