jsql-neo 5.4.2 → 6.0.0-beta2

package/nativesrc/jsql-neo-core/src/storage/wal.rs

@@ -1,13 +1,20 @@
+//! 预写日志（WAL）。
+//!
+//! 记录格式：`[len: u64 LE][crc32: u32 LE][bincode(WalEntry)]`，头部与载荷一次性写出，
+//! 每条记录带 CRC 校验。重放时遇到截断或校验失败的记录即停止，把它视为「未完整落盘」，
+//! 这样即使进程在写一半时被杀，也不会把半截数据当成有效记录。
+
 use serde::{Deserialize, Serialize};
-use std::io::{Read, Write};
+use std::io::{Read, Seek, SeekFrom, Write};
 
-#[derive(Debug, Clone, Serialize, Deserialize)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[repr(u8)]
 pub enum WalOp {
-    CreateTable,
-    DropTable,
-    Insert,
-    Update,
-    Remove,
+    CreateTable = 0,
+    DropTable = 1,
+    Insert = 2,
+    Update = 3,
+    Remove = 4,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -17,40 +24,133 @@ pub struct WalEntry {
     pub data: Vec<u8>,
 }
 
+/// 记录头长度：len(u64) + crc32(u32)
+const HEADER_LEN: usize = 12;
+/// 单条载荷上限，避免损坏的长度字段触发超大内存分配
+const MAX_PAYLOAD: usize = 64 * 1024 * 1024;
+
+fn io_err<E: std::fmt::Display>(e: E) -> std::io::Error {
+    std::io::Error::new(std::io::ErrorKind::Other, e.to_string())
+}
+
+/// 顺序重放，回调所有 `tx_id > after_tx` 的完整记录。
+/// 返回成功解析的字节数（尾部残缺部分不计入）。
+fn replay<R: Read>(reader: &mut R, after_tx: u64, mut on_entry: impl FnMut(WalEntry)) -> std::io::Result<u64> {
+    let mut valid = 0u64;
+    let mut header = [0u8; HEADER_LEN];
+    loop {
+        if reader.read_exact(&mut header).is_err() {
+            break;
+        }
+        let len = u64::from_le_bytes(header[..8].try_into().unwrap()) as usize;
+        let crc = u32::from_le_bytes(header[8..].try_into().unwrap());
+        if len == 0 || len > MAX_PAYLOAD {
+            break;
+        }
+        let mut body = vec![0u8; len];
+        if reader.read_exact(&mut body).is_err() {
+            break;
+        }
+        if crc32fast::hash(&body) != crc {
+            break;
+        }
+        match bincode::deserialize::<WalEntry>(&body) {
+            Ok(entry) => {
+                valid += (HEADER_LEN + len) as u64;
+                if entry.tx_id > after_tx {
+                    on_entry(entry);
+                }
+            }
+            Err(_) => break,
+        }
+    }
+    Ok(valid)
+}
+
 pub struct WalWriter {
-    path: String,
-    file: Option<std::fs::File>,
+    file: std::fs::File,
     next_tx: u64,
+    pending: u32,
+    sync_interval: u32,
+    buf: Vec<u8>,
 }
 
 impl WalWriter {
     pub fn open(path: &str) -> std::io::Result<Self> {
-        let file = std::fs::OpenOptions::new()
+        let mut file = std::fs::OpenOptions::new()
             .create(true)
+            .read(true)
             .append(true)
             .open(path)?;
-        Ok(Self { path: path.to_string(), file: Some(file), next_tx: 1 })
+        // 从已有文件恢复 next_tx：否则重启后 tx_id 会从 1 重新开始，
+        // 与历史记录撞号，`replay_after` 的增量语义就废了。
+        let mut max_tx = 0u64;
+        replay(&mut file, 0, |e| {
+            if e.tx_id > max_tx {
+                max_tx = e.tx_id;
+            }
+        })?;
+        file.seek(SeekFrom::End(0))?;
+        Ok(Self {
+            file,
+            next_tx: max_tx + 1,
+            pending: 0,
+            sync_interval: 1,
+            buf: Vec::with_capacity(256),
+        })
+    }
+
+    /// 每写多少条记录 fsync 一次。1 = 每条都落盘（默认，最安全）；调大换取吞吐。
+    pub fn set_sync_interval(&mut self, n: u32) {
+        self.sync_interval = n.max(1);
+    }
+
+    pub fn next_tx(&self) -> u64 {
+        self.next_tx
     }
 
-    #[allow(dead_code)]
     pub fn append(&mut self, op: WalOp, data: Vec<u8>) -> std::io::Result<u64> {
         let tx_id = self.next_tx;
+        let body = bincode::serialize(&WalEntry { tx_id, op, data }).map_err(io_err)?;
+        if body.len() > MAX_PAYLOAD {
+            return Err(std::io::Error::new(
+                std::io::ErrorKind::InvalidInput,
+                "wal record too large",
+            ));
+        }
+        // 头 + 载荷一次 write_all，避免两次写之间被打断留下半截头部
+        self.buf.clear();
+        self.buf.extend_from_slice(&(body.len() as u64).to_le_bytes());
+        self.buf.extend_from_slice(&crc32fast::hash(&body).to_le_bytes());
+        self.buf.extend_from_slice(&body);
+        self.file.write_all(&self.buf)?;
+
         self.next_tx += 1;
-        let entry = WalEntry { tx_id, op, data };
-        let buf = bincode::serialize(&entry).map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?;
-        if let Some(ref mut file) = self.file {
-            let len = buf.len() as u64;
-            file.write_all(&len.to_le_bytes())?;
-            file.write_all(&buf)?;
-            file.flush()?;
+        self.pending += 1;
+        if self.pending >= self.sync_interval {
+            self.sync()?;
         }
         Ok(tx_id)
     }
 
+    /// 真正落盘。`File::flush` 是空操作，WAL 必须走 `sync_data` 才算持久化。
+    pub fn sync(&mut self) -> std::io::Result<()> {
+        self.file.sync_data()?;
+        self.pending = 0;
+        Ok(())
+    }
+
+    /// `sync` 的别名
     pub fn flush(&mut self) -> std::io::Result<()> {
-        if let Some(ref mut file) = self.file {
-            file.flush()?;
-        }
+        self.sync()
+    }
+
+    /// 清空日志（快照落盘后调用），避免 WAL 无限增长
+    pub fn truncate(&mut self) -> std::io::Result<()> {
+        self.file.set_len(0)?;
+        self.file.seek(SeekFrom::Start(0))?;
+        self.file.sync_data()?;
+        self.pending = 0;
         Ok(())
     }
 }
@@ -64,23 +164,136 @@ impl WalReader {
         Self { path: path.to_string() }
     }
 
-    pub fn replay_after(&self, _after_tx: u64) -> std::io::Result<Vec<WalEntry>> {
+    /// 增量重放：只返回 `tx_id > after_tx` 的记录。
+    pub fn replay_after(&self, after_tx: u64) -> std::io::Result<Vec<WalEntry>> {
         let file = match std::fs::File::open(&self.path) {
             Ok(f) => f,
             Err(_) => return Ok(Vec::new()),
         };
         let mut reader = std::io::BufReader::new(file);
         let mut entries = Vec::new();
-        loop {
-            let mut len_buf = [0u8; 8];
-            if reader.read_exact(&mut len_buf).is_err() { break; }
-            let len = u64::from_le_bytes(len_buf) as usize;
-            let mut buf = vec![0u8; len];
-            if reader.read_exact(&mut buf).is_err() { break; }
-            if let Ok(entry) = bincode::deserialize::<WalEntry>(&buf) {
-                entries.push(entry);
+        replay(&mut reader, after_tx, |e| entries.push(e))?;
+        Ok(entries)
+    }
+
+    /// 重放全部记录
+    pub fn replay_all(&self) -> std::io::Result<Vec<WalEntry>> {
+        self.replay_after(0)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn tmp_path(tag: &str) -> String {
+        let mut p = std::env::temp_dir();
+        p.push(format!("jsql-wal-{}-{}.log", tag, std::process::id()));
+        let s = p.to_string_lossy().to_string();
+        let _ = std::fs::remove_file(&s);
+        s
+    }
+
+    #[test]
+    fn append_replay_roundtrip() {
+        let path = tmp_path("rt");
+        {
+            let mut w = WalWriter::open(&path).unwrap();
+            for i in 0..5u8 {
+                assert_eq!(w.append(WalOp::Insert, vec![i]).unwrap(), i as u64 + 1);
             }
+            w.sync().unwrap();
         }
-        Ok(entries)
+        let entries = WalReader::open(&path).replay_all().unwrap();
+        assert_eq!(entries.len(), 5);
+        assert_eq!(entries[3].tx_id, 4);
+        assert_eq!(entries[3].op, WalOp::Insert);
+        assert_eq!(entries[3].data, vec![3u8]);
+        let _ = std::fs::remove_file(&path);
+    }
+
+    #[test]
+    fn replay_after_is_incremental() {
+        let path = tmp_path("inc");
+        {
+            let mut w = WalWriter::open(&path).unwrap();
+            for i in 0..5u8 {
+                w.append(WalOp::Update, vec![i]).unwrap();
+            }
+            w.sync().unwrap();
+        }
+        let tail = WalReader::open(&path).replay_after(3).unwrap();
+        assert_eq!(tail.len(), 2);
+        assert!(tail.iter().all(|e| e.tx_id > 3));
+        let _ = std::fs::remove_file(&path);
+    }
+
+    #[test]
+    fn next_tx_is_recovered_after_reopen() {
+        let path = tmp_path("recover");
+        {
+            let mut w = WalWriter::open(&path).unwrap();
+            w.append(WalOp::Insert, vec![1]).unwrap();
+            w.append(WalOp::Insert, vec![2]).unwrap();
+            w.sync().unwrap();
+        }
+        let mut w = WalWriter::open(&path).unwrap();
+        assert_eq!(w.next_tx(), 3, "tx_id 必须接着历史最大值，不能回退到 1");
+        assert_eq!(w.append(WalOp::Remove, vec![3]).unwrap(), 3);
+        assert_eq!(WalReader::open(&path).replay_all().unwrap().len(), 3);
+        let _ = std::fs::remove_file(&path);
+    }
+
+    #[test]
+    fn truncated_tail_is_dropped() {
+        let path = tmp_path("torn");
+        {
+            let mut w = WalWriter::open(&path).unwrap();
+            w.append(WalOp::Insert, vec![7]).unwrap();
+            w.append(WalOp::Insert, vec![8]).unwrap();
+            w.sync().unwrap();
+        }
+        // 模拟写一半被杀：尾部只剩半截垃圾
+        {
+            use std::io::Write as _;
+            let mut f = std::fs::OpenOptions::new().append(true).open(&path).unwrap();
+            f.write_all(&[9u8, 9, 9]).unwrap();
+            f.sync_all().unwrap();
+        }
+        assert_eq!(WalReader::open(&path).replay_all().unwrap().len(), 2);
+        let _ = std::fs::remove_file(&path);
+    }
+
+    #[test]
+    fn crc_mismatch_stops_replay() {
+        let path = tmp_path("crc");
+        {
+            let mut w = WalWriter::open(&path).unwrap();
+            w.append(WalOp::Insert, vec![1]).unwrap();
+            w.append(WalOp::Insert, vec![2]).unwrap();
+            w.sync().unwrap();
+        }
+        let len = std::fs::metadata(&path).unwrap().len();
+        {
+            use std::io::{Seek, SeekFrom, Write as _};
+            let mut f = std::fs::OpenOptions::new().write(true).open(&path).unwrap();
+            f.seek(SeekFrom::Start(len - 1)).unwrap();
+            f.write_all(&[0xFF]).unwrap();
+            f.sync_all().unwrap();
+        }
+        assert_eq!(WalReader::open(&path).replay_all().unwrap().len(), 1, "校验失败记录应丢弃");
+        let _ = std::fs::remove_file(&path);
+    }
+
+    #[test]
+    fn truncate_resets_log_but_keeps_tx_monotonic() {
+        let path = tmp_path("trunc");
+        let mut w = WalWriter::open(&path).unwrap();
+        w.append(WalOp::Insert, vec![1]).unwrap();
+        w.truncate().unwrap();
+        assert_eq!(WalReader::open(&path).replay_all().unwrap().len(), 0);
+        // tx_id 不能回落：否则消费方持有的水位线（replay_after(T)）会看不到新记录
+        assert_eq!(w.append(WalOp::DropTable, vec![]).unwrap(), 2);
+        let _ = std::fs::remove_file(&path);
     }
 }

package/nativesrc/jsql-neo-native/Cargo.lock

@@ -205,12 +205,13 @@ dependencies = [
 
 [[package]]
 name = "jsql-neo-core"
-version = "0.1.0"
+version = "5.6.0"
 dependencies = [
  "bincode",
  "chrono",
  "crc32fast",
  "itoa",
+ "regex-lite",
  "rustc-hash",
  "ryu",
  "serde",
@@ -219,7 +220,7 @@ dependencies = [
 
 [[package]]
 name = "jsql-neo-native"
-version = "0.1.0"
+version = "5.6.0"
 dependencies = [
  "jsql-neo-core",
  "napi",
@@ -375,6 +376,12 @@ dependencies = [
  "regex-syntax",
 ]
 
+[[package]]
+name = "regex-lite"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973"
+
 [[package]]
 name = "regex-syntax"
 version = "0.8.11"

package/nativesrc/jsql-neo-native/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "jsql-neo-native"
-version = "5.4.0"
+version = "6.0.0"
 edition = "2021"
 
 [lib]
@@ -14,3 +14,11 @@ serde_json = "1"
 
 [build-dependencies]
 napi-build = "2"
+
+# 体积更小 + 运行更快：全图 LTO 去掉跨 crate 死代码，单 codegen-unit 换取更好的内联与优化，
+# strip 去掉符号表（不影响 napi 动态导出符号与 dlopen 加载）。
+[profile.release]
+opt-level = 3
+lto = "fat"
+codegen-units = 1
+strip = "symbols"

package/nativesrc/jsql-neo-native/src/lib.rs

@@ -111,6 +111,18 @@ pub fn jsql_create_table(name: String, schema_json: String) -> String {
     })
 }
 
+#[napi]
+pub fn jsql_add_column(table: String, name: String, fs_json: String) -> String {
+    let fs: FieldSchema = match serde_json::from_str(&fs_json) {
+        Ok(f) => f,
+        Err(e) => return format!(r#"{{"ok":false,"error":"invalid field schema: {}"}}"#, e),
+    };
+    with_engine_mut(|eng| match eng.add_column(&table, &name, fs) {
+        Ok(()) => r#"{"ok":true}"#.to_string(),
+        Err(e) => format!(r#"{{"ok":false,"error":"{}"}}"#, e),
+    })
+}
+
 #[napi]
 pub fn jsql_drop_table(name: String) -> String {
     with_engine_mut(|eng| match eng.drop_table(&name) {
@@ -152,8 +164,8 @@ pub fn jsql_find(table: String, filter_json: String, limit: i32, offset: i32) ->
     with_engine_ro(|eng| {
         let filter: Option<HashMap<String, serde_json::Value>> =
             if filter_json.is_empty() { None } else { serde_json::from_str(&filter_json).ok() };
-        match eng.find(&table, &filter, Some(limit as usize), Some(offset as usize), None, &None, &None) {
-            Ok((rows, _, _)) => serde_json::to_string(&rows).unwrap_or_else(|_| "[]".to_string()),
+        match eng.find_json(&table, &filter, Some(limit as usize), Some(offset as usize), None, &None, &None) {
+            Ok((json, _, _)) => json,
             Err(e) => format!(r#"{{"error":"{}"}}"#, e),
         }
     })

package/nativesrc/jsql-neo-wasm/Cargo.lock

@@ -171,13 +171,14 @@ dependencies = [
 
 [[package]]
 name = "jsql-neo-core"
-version = "5.4.0"
+version = "6.0.0"
 dependencies = [
  "bincode",
  "chrono",
  "crc32fast",
  "itoa",
  "js-sys",
+ "regex-lite",
  "rustc-hash",
  "ryu",
  "serde",
@@ -187,7 +188,7 @@ dependencies = [
 
 [[package]]
 name = "jsql-neo-wasm"
-version = "5.4.0"
+version = "6.0.0"
 dependencies = [
  "js-sys",
  "jsql-neo-core",
@@ -252,6 +253,12 @@ dependencies = [
  "proc-macro2",
 ]
 
+[[package]]
+name = "regex-lite"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973"
+
 [[package]]
 name = "rustc-hash"
 version = "2.1.3"

package/nativesrc/jsql-neo-wasm/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "jsql-neo-wasm"
-version = "5.4.0"
+version = "6.0.0"
 edition = "2021"
 
 [lib]
@@ -13,3 +13,10 @@ serde_json = "1"
 
 [dependencies.js-sys]
 version = "0.3"
+
+# wasm 产物：LTO + 单 codegen-unit 显著减小体积，strip 去掉符号表
+[profile.release]
+opt-level = 3
+lto = "fat"
+codegen-units = 1
+strip = "symbols"

package/nativesrc/jsql-neo-wasm/src/lib.rs

@@ -34,6 +34,22 @@ pub fn jsql_create_table(name: &str, schema_json: &str) -> String {
     })
 }
 
+/// 运行期加列：扩展表结构并为已有行回填默认值。
+/// fs_json 为单个 FieldSchema 的 JSON，例如 {"type":"integer","nullable":true,"default":0}
+#[wasm_bindgen]
+pub fn jsql_add_column(table: &str, name: &str, fs_json: &str) -> String {
+    ENGINE.with(|eng| {
+        let fs: FieldSchema = match serde_json::from_str(fs_json) {
+            Ok(f) => f,
+            Err(e) => return format!(r#"{{"ok":false,"error":"invalid field schema: {}"}}"#, e),
+        };
+        match eng.borrow_mut().add_column(table, name, fs) {
+            Ok(()) => r#"{"ok":true}"#.to_string(),
+            Err(e) => format!(r#"{{"ok":false,"error":"{}"}}"#, e),
+        }
+    })
+}
+
 #[wasm_bindgen]
 pub fn jsql_drop_table(name: &str) -> String {
     ENGINE.with(|eng| match eng.borrow_mut().drop_table(name) {
@@ -328,8 +344,8 @@ pub fn jsql_find(table: &str, filter_json: &str, limit: usize, offset: usize) ->
     ENGINE.with(|eng| {
         let filter: Option<std::collections::HashMap<String, serde_json::Value>> =
             if filter_json.is_empty() { None } else { serde_json::from_str(filter_json).ok() };
-        match eng.borrow().find(table, &filter, Some(limit), Some(offset), None, &None, &None) {
-            Ok((rows, _, _)) => serde_json::to_string(&rows).unwrap_or_else(|_| "[]".to_string()),
+        match eng.borrow().find_json(table, &filter, Some(limit), Some(offset), None, &None, &None) {
+            Ok((json, _, _)) => json,
             Err(e) => format!(r#"{{"error":"{}"}}"#, e),
         }
     })

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "jsql-neo",
-  "version": "5.4.2",
+  "version": "6.0.0-beta2",
   "description": "JSQL-NEO — Rust-powered embedded database with WASM, REST API, B-Tree indexes, WAL, crash recovery",
   "main": "index.js",
   "types": "index.d.ts",
@@ -45,11 +45,12 @@
   ],
   "scripts": {
     "postinstall": "node postinstall.js",
-    "test": "node test/smoke.js",
+    "test": "node test/smoke.js && node test/sql-parser.test.js",
+    "test:parser": "node test/sql-parser.test.js",
     "test:btree": "node test/btree.test.js",
     "test:native": "node test/native.test.js",
     "test:regress": "node test/regress-5.1.0.js",
-    "test:all": "node test/smoke.js && node test/native.test.js && node test/wasm.test.js && node test/coverage.js && node test/regress-5.1.0.js && node test/join.test.js && node test/btree.test.js",
+    "test:all": "node test/smoke.js && node test/sql-parser.test.js && node test/native.test.js && node test/wasm.test.js && node test/coverage.js && node test/regress-5.1.0.js && node test/join.test.js && node test/btree.test.js",
     "test:coverage": "c8 --all --include='lib/**' --exclude='lib/mysql_server.js' --exclude='lib/native_client.js' --exclude='lib/wasm_client.js' --exclude='lib/mysql_compat.js' --exclude='lib/nedb_compat.js' --exclude='lib/plugin.js' --reporter=text --reporter=lcov node test/coverage.js",
     "test:orms": "node examples/orms/run-all.js"
   },