dsh-sql 0.1.1 → 0.2.0

package/README.en.md

@@ -63,10 +63,10 @@ sql_exec { sql: UPDATE orders SET status = 'paid' WHERE id = 42 }
 
 ## Safety
 
-- **Read-only whitelist**: sql_query only allows SELECT/PRAGMA/EXPLAIN/SHOW/DESCRIBE/WITH; multi-statement queries are rejected outright
+- **Lexer-grade read-only guard**: sql_query strips strings/comments before validation, then rejects data-modifying CTEs (WITH…DELETE/UPDATE), SELECT INTO, FOR UPDATE/FOR SHARE, PRAGMA assignment, and multi-statement input
 - **Write approval gate**: sql_exec asks for approval by default (mirroring dsh-email's send approval); headless environments without an approval channel are denied
 - **readOnly mode**: lock out writes entirely for production databases
-- **Row clamping**: maxRows capped at 10000, overflow flagged with truncated
+- **Streaming row cap**: SQLite iterator / MySQL stream / PostgreSQL portal all stop at maxRows+1, so large queries are never fully materialized; overflow is flagged with truncated
 - **Identifier validation**: table names restricted to alphanumerics and underscores — no schema injection
 - **Secrets stay out of config**: passwords via `DSH_SQL_PASSWORD_<CONNECTION>` env vars
 
@@ -80,7 +80,7 @@ sql_exec { sql: UPDATE orders SET status = 'paid' WHERE id = 42 }
 
 ```bash
 pnpm install
-pnpm test       # build + 27 tests, including a real SQLite integration suite
+pnpm test       # build + 33 tests, including a real SQLite integration suite
 ```
 
 ## License

package/README.md

@@ -63,10 +63,10 @@ sql_exec { sql: UPDATE orders SET status = 'paid' WHERE id = 42 }
 
 ## 安全设计
 
-- **只读白名单**：sql_query 只放行 SELECT/PRAGMA/EXPLAIN/SHOW/DESCRIBE/WITH，多语句直接拒绝
+- **词法级只读保护**：sql_query 先剥离字符串/注释再校验，拒绝 data-modifying CTE（WITH…DELETE/UPDATE）、SELECT INTO、FOR UPDATE/FOR SHARE、PRAGMA 赋值与多语句
 - **写审批门**：sql_exec 默认弹审批（对齐 dsh-email 的发信审批），headless 环境无审批通道时拒绝执行
 - **readOnly 模式**：生产库可整体禁用写
-- **行数钳制**：maxRows 上限 10000，超量截断并标记 truncated
+- **流式行数钳制**：SQLite 迭代器 / MySQL stream / PostgreSQL portal 都按 maxRows+1 停表，大查询不会全量载入内存，超量标记 truncated
 - **标识符校验**：表名只允许字母/数字/下划线，杜绝 schema 注入
 - **密钥不落配置**：密码支持 `DSH_SQL_PASSWORD_<连接名>` 环境变量
 
@@ -80,7 +80,7 @@ sql_exec { sql: UPDATE orders SET status = 'paid' WHERE id = 42 }
 
 ```bash
 pnpm install
-pnpm test       # 构建 + 27 个测试（含真实 SQLite 集成）
+pnpm test       # 构建 + 33 个测试（含真实 SQLite 集成）
 ```
 
 ## License

package/lib/adapters.d.ts

@@ -16,7 +16,7 @@ export interface DatabaseAdapter {
     engine: 'sqlite' | 'mysql' | 'postgres';
     listTables(): Promise<string[]>;
     describeTable(table: string): Promise<ColumnInfo[]>;
-    query(sql: string): Promise<QueryResult>;
+    query(sql: string, limit?: number): Promise<QueryResult>;
     exec(sql: string): Promise<number>;
     ping(): Promise<void>;
     close(): Promise<void>;

package/lib/adapters.js

@@ -49,9 +49,21 @@ class SqliteAdapter {
             primaryKey: Number(row.pk) === 1,
         }));
     }
-    async query(sql) {
-        const rows = this.db.prepare(sql).all();
-        return rowsToColumns(rows);
+    async query(sql, limit) {
+        const statement = this.db.prepare(sql);
+        if (limit === undefined || limit <= 0) {
+            const rows = statement.all();
+            return rowsToColumns(rows);
+        }
+        const columns = statement.columns().map((column) => column.name);
+        const rows = [];
+        for (const raw of statement.iterate()) {
+            const row = raw;
+            rows.push(columns.map((name) => toValue(row[name])));
+            if (rows.length >= limit)
+                break;
+        }
+        return { columns, rows };
     }
     async exec(sql) {
         const single = sql.replace(/;\s*$/, '').trim();
@@ -98,9 +110,42 @@ class MysqlAdapter {
             primaryKey: String(row.Key ?? '').toUpperCase() === 'PRI',
         }));
     }
-    async query(sql) {
-        const [rows] = await this.pool.query(sql);
-        return rowsToColumns(rows);
+    async query(sql, limit) {
+        if (limit === undefined || limit <= 0) {
+            const [rows] = await this.pool.query(sql);
+            return rowsToColumns(rows);
+        }
+        const corePool = this.pool.pool;
+        return await new Promise((resolve, reject) => {
+            let settled = false;
+            let columns = [];
+            const rows = [];
+            const stream = corePool.query(sql).stream({ highWaterMark: 64 });
+            const finish = () => {
+                if (settled)
+                    return;
+                settled = true;
+                resolve({ columns, rows });
+            };
+            stream.on('fields', (fields) => {
+                columns = fields.map((field) => field.name);
+            });
+            stream.on('result', (row) => {
+                if (columns.length === 0)
+                    columns = Object.keys(row);
+                rows.push(columns.map((name) => toValue(row[name])));
+                if (rows.length >= limit)
+                    stream.destroy();
+            });
+            stream.on('end', finish);
+            stream.on('close', finish);
+            stream.on('error', (error) => {
+                if (settled)
+                    return;
+                settled = true;
+                reject(error instanceof Error ? error : new Error(String(error)));
+            });
+        });
     }
     async exec(sql) {
         const [result] = await this.pool.query(sql);
@@ -133,16 +178,36 @@ class PostgresAdapter {
     }
     async describeTable(table) {
         const name = assertIdentifier(table, '表名');
-        const result = await this.pool.query('SELECT column_name, data_type, is_nullable FROM information_schema.columns WHERE table_name = $1 ORDER BY ordinal_position', [name]);
+        const result = await this.pool.query(`SELECT c.column_name, c.data_type, c.is_nullable,
+              EXISTS (
+                SELECT 1
+                FROM information_schema.table_constraints tc
+                JOIN information_schema.key_column_usage kcu
+                  ON tc.constraint_name = kcu.constraint_name
+                 AND tc.table_schema = kcu.table_schema
+                WHERE tc.table_schema = c.table_schema
+                  AND tc.table_name = c.table_name
+                  AND tc.constraint_type = 'PRIMARY KEY'
+                  AND kcu.column_name = c.column_name
+              ) AS is_primary
+         FROM information_schema.columns c
+        WHERE c.table_schema = 'public'
+          AND c.table_name = $1
+        ORDER BY c.ordinal_position`, [name]);
         return result.rows.map((row) => ({
             name: String(row.column_name),
             type: String(row.data_type ?? ''),
             notNull: String(row.is_nullable) === 'NO',
-            primaryKey: false,
+            primaryKey: row.is_primary === true,
         }));
     }
-    async query(sql) {
-        const result = await this.pool.query(sql);
+    async query(sql, limit) {
+        if (limit === undefined || limit <= 0) {
+            const result = await this.pool.query(sql);
+            const rows = result.rows;
+            return rowsToColumns(rows);
+        }
+        const result = await this.pool.query({ text: sql, values: [], rows: limit });
         const rows = result.rows;
         return rowsToColumns(rows);
     }

package/lib/tools.d.ts

@@ -27,6 +27,8 @@ export interface SqlToolDefinition {
     gate?(exec: unknown, next: () => Promise<unknown>): Promise<unknown>;
     timeoutMs?: number;
 }
+/** 校验只读查询：词法去噪后白名单开头 + 写关键字扫描 + 单语句。 */
+export declare function assertReadQuery(sql: string): string;
 /** 审批执行上下文的最小面。 */
 export interface SqlExecGateContext {
     agent?: unknown;

package/lib/tools.js

@@ -34,16 +34,128 @@ function requiredString(args, key, label) {
 }
 /** 只读语句关键字白名单。 */
 const READ_KEYWORDS = /^(select|pragma|explain|show|describe|desc|with)\b/i;
-/** 校验只读查询：白名单开头 + 拒绝多语句。 */
-function assertReadQuery(sql) {
+/** 去掉字符串、引号标识符与注释，保留真实 SQL 关键字与分号。 */
+function stripSqlNoise(sql) {
+    let out = '';
+    let i = 0;
+    while (i < sql.length) {
+        const ch = sql[i];
+        const next = sql[i + 1];
+        if (ch === '-' && next === '-') {
+            i += 2;
+            while (i < sql.length && sql[i] !== '\n')
+                i += 1;
+            continue;
+        }
+        if (ch === '/' && next === '*') {
+            i += 2;
+            while (i + 1 < sql.length && !(sql[i] === '*' && sql[i + 1] === '/'))
+                i += 1;
+            i += 2;
+            continue;
+        }
+        if (ch === '#' && (i === 0 || /\s/.test(sql[i - 1]))) {
+            if (/^#(?:>>?|-)/.test(sql.slice(i))) {
+                out += ch;
+                i += 1;
+                continue;
+            }
+            i += 1;
+            while (i < sql.length && sql[i] !== '\n')
+                i += 1;
+            continue;
+        }
+        if (ch === "'") {
+            out += ' ';
+            i += 1;
+            while (i < sql.length) {
+                if (sql[i] === "'" && sql[i + 1] === "'") {
+                    i += 2;
+                    continue;
+                }
+                if (sql[i] === "'") {
+                    i += 1;
+                    break;
+                }
+                if (sql[i] === '\\') {
+                    i += 2;
+                    continue;
+                }
+                i += 1;
+            }
+            continue;
+        }
+        if (ch === '"' || ch === '`') {
+            out += ' ';
+            i += 1;
+            while (i < sql.length) {
+                if (sql[i] === ch) {
+                    i += 1;
+                    break;
+                }
+                if (sql[i] === '\\') {
+                    i += 2;
+                    continue;
+                }
+                i += 1;
+            }
+            continue;
+        }
+        if (ch === '[') {
+            out += ' ';
+            i += 1;
+            while (i < sql.length && sql[i] !== ']')
+                i += 1;
+            i += 1;
+            continue;
+        }
+        if (ch === '$') {
+            const dollar = /^\$[A-Za-z_][A-Za-z0-9_]*\$|^\$\$/.exec(sql.slice(i));
+            if (dollar !== null) {
+                out += ' ';
+                i += dollar[0].length;
+                const end = sql.indexOf(dollar[0], i);
+                i = end === -1 ? sql.length : end + dollar[0].length;
+                continue;
+            }
+        }
+        out += ch;
+        i += 1;
+    }
+    return out;
+}
+/** 写操作关键字：出在 SELECT/EXPLAIN/WITH 语句里即拒绝。 */
+const WRITE_KEYWORDS = /\b(insert|update|delete|replace|merge|drop|alter|create|truncate|call|execute|copy|grant|revoke|attach|detach|vacuum|reindex|refresh|set|reset|begin|commit|rollback|savepoint|release|analyze|load_extension)\b/gi;
+/** 校验只读查询：词法去噪后白名单开头 + 写关键字扫描 + 单语句。 */
+export function assertReadQuery(sql) {
     const trimmed = sql.trim();
-    if (!READ_KEYWORDS.test(trimmed)) {
+    const clean = stripSqlNoise(trimmed);
+    const first = /^[a-z]+/i.exec(clean.trim())?.[0]?.toLowerCase() ?? '';
+    if (!READ_KEYWORDS.test(clean.trim())) {
         throw new Error('sql_query 只接受只读语句（SELECT / PRAGMA / EXPLAIN / SHOW / DESCRIBE / WITH）。写操作请用 sql_exec。');
     }
-    const statements = trimmed.split(';').filter((part) => part.trim() !== '');
+    const statements = clean.split(';').filter((part) => part.trim() !== '');
     if (statements.length > 1)
         throw new Error('sql_query 一次只允许一条语句。');
-    return statements[0].trim();
+    const single = statements[0]?.trim() ?? '';
+    if (first === 'pragma') {
+        if (/=/.test(single))
+            throw new Error('sql_query 不接受带赋值参数的 PRAGMA 写操作（如 PRAGMA journal_mode=WAL），请用 sql_exec。');
+    }
+    else if (first === 'show' || first === 'describe' || first === 'desc') {
+        // SHOW / DESCRIBE 本身只读，不再扫写关键字（避免误伤 SHOW CREATE TABLE）。
+    }
+    else {
+        if (/\binto\b/i.test(single))
+            throw new Error('sql_query 不接受 SELECT INTO（写表或 OUTFILE），请用 sql_exec。');
+        if (/\bfor\s+(update|no\s+key\s+update|share|key\s+share)\b/i.test(single))
+            throw new Error('sql_query 不接受 FOR UPDATE/FOR SHARE 行锁查询，请用 sql_exec。');
+        const hit = WRITE_KEYWORDS.exec(single);
+        if (hit !== null) {
+            throw new Error('检测到写操作关键字 ' + hit[0].toUpperCase() + '，sql_query 只接受只读查询。写操作请用 sql_exec。');
+        }
+    }
+    return trimmed.replace(/;+\s*$/, '').trim();
 }
 const querySchema = {
     type: 'object',
@@ -159,7 +271,7 @@ export function buildSqlTools(config) {
     };
     const sqlQuery = {
         name: 'sql_query',
-        description: '执行只读 SQL 查询（SELECT / PRAGMA / EXPLAIN / SHOW / DESCRIBE / WITH）。connection 为连接名（缺省第一个连接）。返回列名与行数据，最多 maxRows 行（超出 truncated=true）。写操作请用 sql_exec。',
+        description: '执行只读 SQL 查询（SELECT / PRAGMA / EXPLAIN / SHOW / DESCRIBE / WITH）。词法级校验会拒绝 data-modifying CTE、SELECT INTO、FOR UPDATE/FOR SHARE、PRAGMA 赋值与多语句。connection 为连接名（缺省第一个连接）。返回列名与行数据，最多 maxRows 行（超出 truncated=true）。写操作请用 sql_exec。',
         parameters: compileParameters({
             sql: { type: 'string', required: true, description: '只读 SQL 语句（必填，单条）。' },
             connection: { type: 'string', description: '连接名（可选，缺省第一个连接）。' },
@@ -182,7 +294,7 @@ export function buildSqlTools(config) {
             const args = asRecord(rawArgs);
             const sql = assertReadQuery(requiredString(args, 'sql', 'SQL 语句'));
             const { adapter, name } = getAdapter(optionalString(args, 'connection'));
-            const result = await adapter.query(sql);
+            const result = await adapter.query(sql, cfg.maxRows + 1);
             const total = result.rows.length;
             const rows = result.rows.slice(0, cfg.maxRows);
             return {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "dsh-sql",
-  "version": "0.1.1",
-  "description": "DSH 工程师级数据库插件：sql_list/query/exec/schema 四工具，SQLite/MySQL/PostgreSQL 三引擎、多连接、只读模式与写操作审批门、行数钳制。",
+  "version": "0.2.0",
+  "description": "DSH 工程师级数据库插件：sql_list/query/exec/schema 四工具，SQLite/MySQL/PostgreSQL 三引擎、多连接、只读模式与写操作审批门、词法级只读保护与流式行数钳制。",
   "type": "module",
   "main": "lib/index.js",
   "types": "lib/index.d.ts",