jsql-neo 6.0.4 → 6.0.5

package/lib/migrate.js

@@ -12,6 +12,15 @@ const path = require('path');
 const { splitStatements, executeSQL } = require('./sql');
 const { parseFieldShorthand } = require('./database');
 
+// 体积护栏（CWE-770）：与 lib/sql.js 的 DEFAULT_MAX_SQL_LENGTH 同风格。
+// 可用 opts.maxInputSize / opts.maxRows 覆盖；传 Infinity 关闭；chunkSize 控制每批 insert 行数。
+const DEFAULT_MAX_CSV_LENGTH = 64 * 1024 * 1024;  // 64 MiB
+const DEFAULT_MAX_JSON_LENGTH = 64 * 1024 * 1024;
+const DEFAULT_MAX_SQL_LENGTH = 64 * 1024 * 1024;  // 与 lib/sql.js 保持一致
+const DEFAULT_MAX_ROWS = 2_000_000;             // 200 万行
+const DEFAULT_MAX_STATEMENTS = 200_000;          // dump 文件最多允许 20 万条语句
+const DEFAULT_CHUNK_SIZE = 1000;
+
 function normalizeSchema(schema) {
   const out = {};
   for (const [name, def] of Object.entries(schema || {})) {
@@ -52,10 +61,31 @@ function parseValue(str, type) {
   return str;
 }
 
-function parseCSV(text) {
+function parseCSV(text, opts = {}) {
+  if (typeof text !== 'string') {
+    throw new Error('parseCSV: csv must be a string');
+  }
+  const maxLen = opts.maxInputSize == null ? DEFAULT_MAX_CSV_LENGTH : opts.maxInputSize;
+  if (typeof maxLen === 'number' && Number.isFinite(maxLen) && text.length > maxLen) {
+    throw new Error(
+      `CSV text too large: ${text.length} chars exceeds limit of ${maxLen} ` +
+      `(raise opts.maxInputSize to allow larger input)`
+    );
+  }
+  const maxRows = opts.maxRows == null ? DEFAULT_MAX_ROWS : opts.maxRows;
+  const isFiniteMaxRows = typeof maxRows === 'number' && Number.isFinite(maxRows);
+
   const rows = [];
+  // 把 `field += c` 改为数组收集中间 join —— 与 splitStatements 同修 O(n²)
+  let fieldBuf = [];
+  const flushField = () => { const s = fieldBuf.join(''); fieldBuf.length = 0; return s; };
   let row = [];
-  let field = '';
+  const flushRow = () => {
+    const r = row;
+    row = [];
+    rows.push(r);
+    return r;
+  };
   let inQuotes = false;
   let i = 0;
   const n = text.length;
@@ -63,32 +93,48 @@ function parseCSV(text) {
     const c = text[i];
     if (inQuotes) {
       if (c === '"') {
-        if (text[i + 1] === '"') { field += '"'; i += 2; continue; }
+        if (text[i + 1] === '"') { fieldBuf.push('"'); i += 2; continue; }
         inQuotes = false;
         i++;
         continue;
       }
-      field += c;
+      fieldBuf.push(c);
       i++;
       continue;
     }
-    if (c === '"' && field === '') { inQuotes = true; i++; continue; }
-    if (c === ',') { row.push(field); field = ''; i++; continue; }
+    if (c === '"' && fieldBuf.length === 0) { inQuotes = true; i++; continue; }
+    if (c === ',') { row.push(flushField()); i++; continue; }
     if (c === '\n' || c === '\r') {
       if (c === '\r' && text[i + 1] === '\n') i++;
-      row.push(field);
-      field = '';
-      if (row.length > 1 || row[0] !== '') rows.push(row);
-      row = [];
+      row.push(flushField());
+      if (row.length > 1 || row[0] !== '') {
+        if (isFiniteMaxRows && rows.length + 1 > maxRows) {
+          throw new Error(
+            `CSV too many rows: ${rows.length + 1} exceeds limit of ${maxRows} ` +
+            `(raise opts.maxRows or split the file)`
+          );
+        }
+        flushRow();
+      } else {
+        row = [];
+      }
       i++;
       continue;
     }
-    field += c;
+    fieldBuf.push(c);
     i++;
   }
-  if (field !== '' || row.length > 0) {
-    row.push(field);
-    if (row.length > 1 || row[0] !== '') rows.push(row);
+  if (fieldBuf.length > 0 || row.length > 0) {
+    row.push(flushField());
+    if (row.length > 1 || row[0] !== '') {
+      if (isFiniteMaxRows && rows.length + 1 > maxRows) {
+        throw new Error(
+          `CSV too many rows: ${rows.length + 1} exceeds limit of ${maxRows} ` +
+          `(raise opts.maxRows or split the file)`
+        );
+      }
+      flushRow();
+    }
   }
   return rows;
 }
@@ -123,7 +169,23 @@ async function exportAllToJSON(engine, tables) {
 
 async function importFromJSON(engine, data, opts = {}) {
   const overwrite = !!(opts && opts.overwrite);
-  let tables = typeof data === 'string' ? JSON.parse(data) : data;
+  let tables;
+  if (typeof data === 'string') {
+    // JSON.parse 吞超长输入也会打爆堆 —— 先加体积护栏，错误带位置
+    const maxLen = opts.maxInputSize == null ? DEFAULT_MAX_JSON_LENGTH : opts.maxInputSize;
+    if (typeof maxLen === 'number' && Number.isFinite(maxLen) && data.length > maxLen) {
+      throw new Error(
+        `JSON text too large: ${data.length} chars exceeds limit of ${maxLen} ` +
+        `(raise opts.maxInputSize to allow larger input)`
+      );
+    }
+    try { tables = JSON.parse(data); }
+    catch (e) {
+      throw new Error(`importFromJSON: invalid JSON: ${e.message}`);
+    }
+  } else {
+    tables = data;
+  }
   // 兼容单表形状：exportTableToJSON 返回 { table, schema, rows }
   if (tables && !Array.isArray(tables) && typeof tables === 'object'
       && tables.table && tables.schema && !tables[tables.table]) {
@@ -142,8 +204,13 @@ async function importFromJSON(engine, data, opts = {}) {
     await engine.createTable(name, normalizeSchema(t.schema));
     created.push(name);
     if (Array.isArray(t.rows) && t.rows.length > 0) {
-      const ids = await engine.insert(name, t.rows.map(r => ({ ...r })));
-      inserted += Array.isArray(ids) ? ids.length : t.rows.length;
+      // 分块 insert —— 避免全量 rows.map 先攒再 insert
+      const chunkSize = opts.chunkSize != null ? opts.chunkSize : DEFAULT_CHUNK_SIZE;
+      for (let i = 0; i < t.rows.length; i += chunkSize) {
+        const chunk = t.rows.slice(i, i + chunkSize).map(r => ({ ...r }));
+        const ids = await engine.insert(name, chunk);
+        inserted += Array.isArray(ids) ? ids.length : chunk.length;
+      }
     }
   }
   return { created, inserted };
@@ -161,7 +228,8 @@ async function exportTableToCSV(engine, table) {
 
 async function importFromCSV(engine, table, csv, opts = {}) {
   const schema = opts.schema || await engine.getTableSchema(table);
-  const rows = parseCSV(csv);
+  // parseCSV 内已有体积/行数护栏，这里传 opts.maxInputSize / opts.maxRows
+  const rows = parseCSV(csv, opts);
   if (rows.length === 0) return { inserted: 0 };
   let columns;
   let start = 0;
@@ -179,24 +247,43 @@ async function importFromCSV(engine, table, csv, opts = {}) {
     }
     await engine.createTable(table, normalizeSchema(schema));
   }
-  const dataRows = [];
-  for (let i = start; i < rows.length; i++) {
-    const row = {};
-    for (let j = 0; j < columns.length; j++) {
-      const type = schema && schema[columns[j]] ? schema[columns[j]].type : 'string';
-      row[columns[j]] = parseValue(rows[i][j], type);
-    }
-    if (row.id === null || row.id === undefined || row.id === '') delete row.id;
-    dataRows.push(row);
+  const chunkSize = opts.chunkSize != null ? opts.chunkSize : DEFAULT_CHUNK_SIZE;
+  let inserted = 0;
+  // 不攒 dataRows 全量，边解析边分块写入 —— 避免 2M 行全在内存里
+  for (let i = start; i < rows.length; i += chunkSize) {
+    const chunk = rows.slice(i, i + chunkSize);
+    const dataRows = chunk.map(raw => {
+      const row = {};
+      for (let j = 0; j < columns.length; j++) {
+        const type = schema && schema[columns[j]] ? schema[columns[j]].type : 'string';
+        row[columns[j]] = parseValue(raw[j], type);
+      }
+      if (row.id === null || row.id === undefined || row.id === '') delete row.id;
+      return row;
+    });
+    const ids = await engine.insert(table, dataRows);
+    inserted += Array.isArray(ids) ? ids.length : dataRows.length;
   }
-  const ids = dataRows.length > 0 ? await engine.insert(table, dataRows) : [];
-  return { inserted: dataRows.length, ids };
+  return { inserted };
 }
 
 /* ---------- mysqldump ---------- */
 
 async function importDump(engine, sqlText, opts = {}) {
-  const statements = splitStatements(sqlText);
+  // splitStatements 自身有体积护栏，但这里要让用户能通过 opts.maxInputSize 覆盖，
+  // 还要额外加「语句数量」护栏 —— 64 MiB 里可能塞几百万条小 INSERT，每条都合法但合起来打爆堆。
+  const maxLen = opts.maxInputSize == null ? DEFAULT_MAX_SQL_LENGTH : opts.maxInputSize;
+  const splitOpts = { maxLength: maxLen };
+  const statements = splitStatements(sqlText, splitOpts);
+
+  const maxStmts = opts.maxStatements == null ? DEFAULT_MAX_STATEMENTS : opts.maxStatements;
+  if (typeof maxStmts === 'number' && Number.isFinite(maxStmts) && statements.length > maxStmts) {
+    throw new Error(
+      `SQL dump too many statements: ${statements.length} exceeds limit of ${maxStmts} ` +
+      `(raise opts.maxStatements or split the dump file)`
+    );
+  }
+
   const created = [];
   let inserted = 0;
   const errors = [];
@@ -221,6 +308,15 @@ async function importDump(engine, sqlText, opts = {}) {
 }
 
 async function importDumpFile(engine, filePath, opts = {}) {
+  // 先看文件大小再决定是否读 —— 避免 fs.readFileSync 把超大 dump 一次性吃进堆
+  const maxLen = opts.maxInputSize == null ? DEFAULT_MAX_SQL_LENGTH : opts.maxInputSize;
+  const stat = fs.statSync(filePath);
+  if (typeof maxLen === 'number' && Number.isFinite(maxLen) && stat.size > maxLen) {
+    throw new Error(
+      `Dump file too large: ${stat.size} bytes exceeds limit of ${maxLen} ` +
+      `(raise opts.maxInputSize or split the file)`
+    );
+  }
   const text = fs.readFileSync(filePath, 'utf8');
   return importDump(engine, text, opts);
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "jsql-neo",
-  "version": "6.0.4",
+  "version": "6.0.5",
   "description": "JSQL-NEO — Rust-powered embedded database with WASM, REST API, B-Tree indexes, WAL, crash recovery",
   "main": "index.js",
   "types": "index.d.ts",