jsql-neo 2.2.0 → 2.2.1

package/index.js

@@ -15,5 +15,6 @@ module.exports = {
     BTree: require('./lib/btree'),
     Cache: require('./lib/cache'),
     JSQL_Error: require('./lib/errors').JSQL_Error,
-    ErrorCodes: require('./lib/errors').ErrorCodes
+    ErrorCodes: require('./lib/errors').ErrorCodes,
+    JSQLFormat: require('./lib/jsql_format')
 };

package/lib/compress_pool.js

@@ -0,0 +1,122 @@
+const { Worker, MessageChannel, receiveMessageOnPort } = require('worker_threads');
+const path = require('path');
+const os = require('os');
+const zlib = require('zlib');
+
+const POOL_SIZE = Math.max(2, Math.min(os.cpus().length - 1, 8));
+
+let pool = null;
+
+class SyncCompressionPool {
+  constructor(size) {
+    this.size = size;
+    this.workers = [];
+    this.ports = [];
+    this.nextId = 1;
+    const wp = path.join(__dirname, 'compress_worker.js');
+
+    for (let i = 0; i < size; i++) {
+      const { port1, port2 } = new MessageChannel();
+      const worker = new Worker(wp);
+      worker.postMessage({ type: 'init', port: port2 }, [port2]);
+      worker.unref();
+      this.workers.push({ worker, port: port1 });
+      this.ports.push(port1);
+    }
+  }
+
+  _getIdle() {
+    for (const p of this.ports) {
+      if (!p._busy) return p;
+    }
+    return null;
+  }
+
+  _copyBuffer(data) {
+    const src = Buffer.isBuffer(data) ? data : Buffer.from(data);
+    const ab = new ArrayBuffer(src.length);
+    const dst = Buffer.from(ab);
+    src.copy(dst);
+    return dst;
+  }
+
+  _sendMsg(port, msg, data) {
+    const copy = this._copyBuffer(data);
+    port._busy = true;
+    port.postMessage({ ...msg, data: copy.buffer, level: msg.level || 1 }, [copy.buffer]);
+  }
+
+  _waitResult(port) {
+    let r = null;
+    while (!r) { r = receiveMessageOnPort(port); }
+    port._busy = false;
+    if (r.message.error) throw new Error(r.message.error);
+    return r.message;
+  }
+
+  compressSync(data, level = 1) {
+    const port = this._getIdle();
+    if (!port) {
+      return { result: zlib.gzipSync(Buffer.isBuffer(data) ? data : Buffer.from(data), { level }).buffer };
+    }
+    const id = this.nextId++;
+    this._sendMsg(port, { type: 'compress', id, level }, data);
+    return this._waitResult(port);
+  }
+
+  decompressSync(data) {
+    const port = this._getIdle();
+    if (!port) {
+      return { result: zlib.gunzipSync(Buffer.isBuffer(data) ? data : Buffer.from(data)).buffer };
+    }
+    const id = this.nextId++;
+    this._sendMsg(port, { type: 'decompress', id }, data);
+    return this._waitResult(port);
+  }
+
+  compressBatchSync(blocks, level = 1) {
+    const n = blocks.length;
+    const results = new Array(n);
+    let submitted = 0;
+    let completed = 0;
+
+    const trySubmit = () => {
+      while (submitted < n) {
+        const port = this._getIdle();
+        if (!port) break;
+        const idx = submitted++;
+        port._idx = idx;
+        port._busy = true;
+        const copy = this._copyBuffer(blocks[idx]);
+        port.postMessage({ type: 'compress', id: this.nextId++, data: copy.buffer, level }, [copy.buffer]);
+      }
+    };
+
+    trySubmit();
+    while (completed < n) {
+      for (const port of this.ports) {
+        if (!port._busy) continue;
+        const msg = receiveMessageOnPort(port);
+        if (msg) {
+          results[port._idx] = msg.message;
+          port._busy = false;
+          completed++;
+          trySubmit();
+        }
+      }
+    }
+    return results;
+  }
+
+  terminate() {
+    for (const { worker } of this.workers) worker.terminate();
+    pool = null;
+  }
+}
+
+function getPool() {
+  if (!pool) pool = new SyncCompressionPool(POOL_SIZE);
+  return pool;
+}
+
+module.exports = { getPool, SyncCompressionPool };

package/lib/compress_worker.js

@@ -0,0 +1,32 @@
+const { parentPort } = require('worker_threads');
+const zlib = require('zlib');
+
+let commPort = null;
+
+parentPort.on('message', (msg) => {
+  if (msg.type === 'init' && msg.port) {
+    commPort = msg.port;
+    commPort.on('message', (task) => {
+      try {
+        if (task.type === 'compress') {
+          const buf = Buffer.from(task.data);
+          const result = zlib.gzipSync(buf, { level: task.level || 1 });
+          const ab = new ArrayBuffer(result.length);
+          const view = new Uint8Array(ab);
+          for (let i = 0; i < result.length; i++) view[i] = result[i];
+          commPort.postMessage({ id: task.id, result: ab, len: result.length }, [ab]);
+        } else if (task.type === 'decompress') {
+          const buf = Buffer.from(task.data);
+          const result = zlib.gunzipSync(buf);
+          const ab = new ArrayBuffer(result.length);
+          const view = new Uint8Array(ab);
+          for (let i = 0; i < result.length; i++) view[i] = result[i];
+          commPort.postMessage({ id: task.id, result: ab, len: result.length }, [ab]);
+        }
+      } catch (e) {
+        commPort.postMessage({ id: task.id, error: e.message });
+      }
+    });
+    commPort.unref();
+  }
+});

package/lib/database.js

@@ -9,6 +9,7 @@ const path = require('path');
 const crypto = require('crypto');
 const Table = require('./table');
 const { createError } = require('./errors');
+const JSQLFormat = require('./jsql_format');
 
 class Database {
     /**
@@ -57,6 +58,10 @@ class Database {
         // 事务隔离
         this._isolationLevel = options.isolationLevel || 'READ_COMMITTED';
 
+        // JSql 格式 (binary block-based)
+        this._jsqlMode = this._filePath && this._filePath.endsWith('.jsql');
+        this._jsqlFormat = this._jsqlMode ? new JSQLFormat(this._filePath) : null;
+
         // 文件锁
         this._fileLockEnabled = options.fileLock === true && !this._memoryMode;
         this._lockFd = null;
@@ -898,6 +903,20 @@ class Database {
 
         if (this._memoryMode) return;
 
+        if (this._jsqlMode) {
+            const all = {};
+            for (const [name, table] of Object.entries(this._tables)) {
+                if (name === '__meta__') continue;
+                const pkField = Object.keys(table._schema).find(f => table._schema[f].primaryKey);
+                all[name] = { schema: table._schema, rows: table._rows, pkField };
+            }
+            this._jsqlFormat.saveAll(all);
+            this._jsqlFormat._close();
+            this._dirty = false;
+            this._checkpoint();
+            return;
+        }
+
         const data = { __schema__: {}, __meta__: { version: this._getCurrentVersion() } };
         for (const [name, table] of Object.entries(this._tables)) {
             if (name === '__meta__') continue;
@@ -917,6 +936,18 @@ class Database {
 
     _load() {
         try {
+        if (this._jsqlMode) {
+            this._jsqlFormat._open();
+            for (const [name, t] of this._jsqlFormat.tables) {
+                if (!this._tables[name]) this.createTable(name, t.schema);
+                if (t.rowCount > 0) {
+                    const { rows } = this._jsqlFormat.readTableSync(name);
+                    this._tables[name]._loadRows(rows);
+                }
+            }
+            return;
+        }
+
             const json = this._readFile(this._filePath);
             const data = JSON.parse(json);
 
@@ -964,6 +995,7 @@ class Database {
         if (this._dirty && !this._memoryMode) {
             this.save();
         }
+        if (this._jsqlFormat) this._jsqlFormat._close();
         this._releaseLock();
         this._tables = {};
     }

package/lib/jsql_format.js

@@ -0,0 +1,478 @@
+const zlib = require('zlib');
+const crypto = require('crypto');
+const fs = require('fs');
+
+
+const SZ = {
+  SUPER: 4096,
+  HEADER: 64,
+  BLOCK: 4096,
+  DATA: 4096 - 64
+};
+
+const BT = { FREE: 0, DATA: 1, IDX: 2 };
+
+const TYPE = { NUL:0, BOL:1, I32:2, F64:3, STR:4, BUF:5, ARR:6, OBJ:7 };
+
+function tag(v) {
+  if (v===null||v===undefined) return TYPE.NUL;
+  if (typeof v==='boolean') return TYPE.BOL;
+  if (typeof v==='number') return Number.isInteger(v)?TYPE.I32:TYPE.F64;
+  if (typeof v==='string') return TYPE.STR;
+  if (v instanceof Buffer) return TYPE.BUF;
+  return Array.isArray(v)?TYPE.ARR:TYPE.OBJ;
+}
+
+function enc(v) {
+  const t = tag(v);
+  switch (t) {
+    case TYPE.NUL: return Buffer.from([t]);
+    case TYPE.BOL: return Buffer.from([t, v?1:0]);
+    case TYPE.I32: { const b=Buffer.alloc(5); b[0]=t; b.writeInt32LE(v,1); return b; }
+    case TYPE.F64: { const b=Buffer.alloc(9); b[0]=t; b.writeDoubleLE(v,1); return b; }
+    case TYPE.STR: { const s=Buffer.from(v,'utf8'); const b=Buffer.alloc(5+s.length); b[0]=t; b.writeUInt32LE(s.length,1); s.copy(b,5); return b; }
+    case TYPE.BUF: { const b=Buffer.alloc(5+v.length); b[0]=t; b.writeUInt32LE(v.length,1); v.copy(b,5); return b; }
+    case TYPE.ARR: { const p=[Buffer.from([t]),enc(v.length)]; for(const i of v) p.push(enc(i)); return Buffer.concat(p); }
+    case TYPE.OBJ: { const k=Object.keys(v); const p=[Buffer.from([t]),enc(k.length)]; for(const f of k){p.push(enc(f));p.push(enc(v[f]));} return Buffer.concat(p); }
+  }
+}
+
+function dec(buf, off) {
+  const t = buf[off++];
+  switch (t) {
+    case TYPE.NUL: return {v:null,off};
+    case TYPE.BOL: return {v:buf[off]!==0,off:off+1};
+    case TYPE.I32: return {v:buf.readInt32LE(off),off:off+4};
+    case TYPE.F64: return {v:buf.readDoubleLE(off),off:off+8};
+    case TYPE.STR: { const l=buf.readUInt32LE(off);off+=4; return {v:buf.toString('utf8',off,off+l),off:off+l}; }
+    case TYPE.BUF: { const l=buf.readUInt32LE(off);off+=4; return {v:Buffer.from(buf.subarray(off,off+l)),off:off+l}; }
+    case TYPE.ARR: { let r=dec(buf,off);off=r.off; const l=r.v,a=new Array(l); for(let i=0;i<l;i++){r=dec(buf,off);a[i]=r.v;off=r.off;} return {v:a,off}; }
+    case TYPE.OBJ: { let r=dec(buf,off);off=r.off; const l=r.v,o={}; for(let i=0;i<l;i++){r=dec(buf,off);const k=r.v;off=r.off;r=dec(buf,off);o[k]=r.v;off=r.off;} return {v:o,off}; }
+    default: throw new Error('bad type '+t);
+  }
+}
+
+// Pre-encoded field names for faster row encoding
+function buildFieldEncoders(schema) {
+  const fields = Object.keys(schema).filter(f => f !== '_softDelete');
+  return fields.map(f => enc(f));
+}
+
+function encRowFast(row, fieldEncoders) {
+  // Estimate: 4(len) + sum of field values
+  // fieldEncoders includes pre-encoded field names
+  let size = 4;
+  for (let i = 0; i < fieldEncoders.length; i++) {
+    size += fieldEncoders[i].length;
+    const v = row[Object.keys(row)[i] || '']; // fallback
+  }
+  // Just use the buffer approach
+  const chunks = [Buffer.alloc(4)];
+  for (let i = 0; i < fieldEncoders.length; i++) {
+    chunks.push(fieldEncoders[i], enc(row[Object.keys(row)[i]]));
+  }
+  const data = Buffer.concat(chunks);
+  data.writeUInt32LE(data.length - 4, 0);
+  return data;
+}
+
+function writeEnc(buf, off, v) {
+  if (v === null || v === undefined) { buf[off] = 0; return off + 1; }
+  if (typeof v === 'boolean') { buf[off] = 1; buf[off+1] = v ? 1 : 0; return off + 2; }
+  if (typeof v === 'number') {
+    if (Number.isInteger(v)) { buf[off] = 2; buf.writeInt32LE(v, off+1); return off + 5; }
+    buf[off] = 3; buf.writeDoubleLE(v, off+1); return off + 9;
+  }
+  if (typeof v === 'string') {
+    buf[off] = 4;
+    const encoded = Buffer.from(v, 'utf8');
+    buf.writeUInt32LE(encoded.length, off+1);
+    encoded.copy(buf, off+5);
+    return off + 5 + encoded.length;
+  }
+  const e = enc(v);
+  e.copy(buf, off);
+  return off + e.length;
+}
+
+function encRowInPlace(row, encoders, scratch, offStart) {
+  let off = offStart + 4;
+  for (let i = 0; i < encoders.length; i++) {
+    encoders[i].nameEnc.copy(scratch, off); off += encoders[i].nameEnc.length;
+    off = writeEnc(scratch, off, row[encoders[i].name]);
+  }
+  scratch.writeUInt32LE(off - offStart - 4, offStart);
+  return off;
+}
+
+// Build reusable field encoders from schema
+function buildEncoders(schema) {
+  return Object.keys(schema).filter(f => f !== '_softDelete').map(name => ({ name, nameEnc: enc(name) }));
+}
+
+const encRow = encRowInPlace;
+
+function decRow(buf, off) {
+  const l = buf.readUInt32LE(off); off += 4;
+  const end = off + l;
+  const row = {};
+  while (off < end) {
+    const r1 = dec(buf, off); const f = r1.v; off = r1.off;
+    const r2 = dec(buf, off); row[f] = r2.v; off = r2.off;
+  }
+  return { row, off };
+}
+
+function hashPK(val) {
+  let h = 0;
+  const s = String(val);
+  for (let i = 0; i < s.length; i++) h = (Math.imul(h,31) + s.charCodeAt(i)) | 0;
+  return h >>> 0;
+}
+
+function mkHeader(type, seq, tableId) {
+  const h = Buffer.alloc(SZ.HEADER);
+  h.writeUInt32LE(seq, 32);
+  h[36] = type;
+  h[37] = tableId || 0;
+  return h;
+}
+
+function finalize(buf) {
+  const h = crypto.createHash('sha256').update(buf.subarray(32)).digest();
+  h.copy(buf, 0);
+}
+
+function verifyHash(buf) {
+  const s = buf.slice(0, 32);
+  const c = crypto.createHash('sha256').update(buf.subarray(32)).digest();
+  return s.equals(c);
+}
+
+class JSQLFormat {
+  constructor(path) {
+    this.path = path;
+    this.fd = null;
+    this.tables = new Map();
+    this.pkIdx = new Map();
+    this.nBlocks = 0;
+    this.dirty = false;
+    this.open = false;
+    this._nextTableId = 1;
+  }
+
+  _open() {
+    if (this.fd) return;
+    this.open = true;
+    if (fs.existsSync(this.path)) {
+      this.fd = fs.openSync(this.path, 'r+');
+      this._load();
+    } else {
+      this.fd = fs.openSync(this.path, 'w+');
+      this._init();
+    }
+  }
+
+  _close() {
+    if (!this.fd) return;
+    if (this.dirty) this._flush();
+    fs.closeSync(this.fd);
+    this.fd = null;
+    this.open = false;
+  }
+
+  _init() {
+    const sb = Buffer.alloc(SZ.SUPER);
+    sb.write('JSQL', 0, 'utf8');
+    // block count = superblock only
+    this.nBlocks = 1;
+    fs.writeSync(this.fd, sb, 0, SZ.SUPER, 0);
+    fs.ftruncateSync(this.fd, SZ.SUPER);
+  }
+
+  _load() {
+    const sb = Buffer.alloc(SZ.SUPER);
+    fs.readSync(this.fd, sb, 0, SZ.SUPER, 0);
+    const magic = sb.toString('utf8', 0, 4);
+    if (magic !== 'JSQL') throw new Error('not a .jsql file');
+
+    const tCount = sb.readUInt16LE(4);
+    let off = 6;
+    for (let i = 0; i < tCount; i++) {
+      const nLen = sb[off]; off += 1;
+      const name = sb.toString('utf8', off, off + nLen); off += nLen;
+      const sLen = sb.readUInt32LE(off); off += 4;
+      const rCnt = sb.readUInt32LE(off); off += 4;
+      const pkB = sb.readUInt32LE(off); off += 4;
+      const dB = sb.readUInt32LE(off); off += 4;
+      const tableId = sb.readUInt16LE(off); off += 2;
+      const schema = JSON.parse(sb.toString('utf8', off, off + sLen)); off += sLen;
+      this.tables.set(name, { schema, rowCount: rCnt, pkBlock: pkB, dataBlock: dB, tableId });
+      this.pkIdx.set(name, this._loadIdx(pkB));
+      if ((this.tables.get(name).tableId || 0) >= this._nextTableId)
+        this._nextTableId = (this.tables.get(name).tableId || 0) + 1;
+    }
+
+    this.nBlocks = Math.max(1, Math.ceil(fs.fstatSync(this.fd).size / SZ.BLOCK));
+  }
+
+  _saveIdx(name) {
+    const entries = this.pkIdx.get(name) || [];
+    const maxPerBlock = Math.floor((SZ.DATA - 4) / 12);
+    let firstBlock = 0, prevBlock = 0;
+
+    for (let chunk = 0; chunk < entries.length; chunk += maxPerBlock) {
+      const chunkEnd = Math.min(chunk + maxPerBlock, entries.length);
+      const buf = Buffer.alloc(SZ.BLOCK);
+      mkHeader(BT.IDX, chunk).copy(buf);
+      let off = SZ.HEADER;
+
+      const cnt = chunkEnd - chunk;
+      buf.writeUInt32LE(cnt, off); off += 4;
+      for (let i = chunk; i < chunkEnd; i++) {
+        const e = entries[i];
+        buf.writeUInt32LE(e.hash, off); off += 4;
+        buf.writeUInt32LE(e.block, off); off += 4;
+        buf.writeUInt32LE(e.off, off); off += 4;
+      }
+
+      const bid = this._alloc();
+      if (chunk === 0) firstBlock = bid;
+      if (prevBlock) buf.writeUInt32LE(bid, 52); // next block pointer
+      buf.writeUInt32LE(prevBlock, 56); // prev block pointer
+
+      // total entry count in first block
+      if (chunk === 0) buf.writeUInt32LE(entries.length, 48);
+
+      finalize(buf);
+      fs.writeSync(this.fd, buf, 0, SZ.BLOCK, bid * SZ.BLOCK);
+      prevBlock = bid;
+    }
+
+    const t = this.tables.get(name);
+    if (t) t.pkBlock = firstBlock;
+    return firstBlock;
+  }
+
+  _loadIdx(blockId) {
+    if (blockId < 1) return [];
+    const all = [];
+    let bid = blockId;
+    while (bid) {
+      const buf = Buffer.alloc(SZ.BLOCK);
+      fs.readSync(this.fd, buf, 0, SZ.BLOCK, bid * SZ.BLOCK);
+      if (!verifyHash(buf)) throw new Error('corrupt idx block ' + bid);
+      let off = SZ.HEADER;
+      const cnt = buf.readUInt32LE(off); off += 4;
+      for (let i = 0; i < cnt; i++) {
+        const hash = buf.readUInt32LE(off); off += 4;
+        const block = buf.readUInt32LE(off); off += 4;
+        const dataOff = buf.readUInt32LE(off); off += 4;
+        all.push({ hash, block, off: dataOff });
+      }
+      bid = buf.readUInt32LE(52);
+    }
+    return all;
+  }
+
+  _alloc() { return this.nBlocks++; }
+
+  _flush() {
+    for (const name of this.tables.keys()) this._saveIdx(name);
+    this._saveSuper();
+    this.dirty = false;
+  }
+
+  _saveSuper() {
+    const sb = Buffer.alloc(SZ.SUPER);
+    sb.write('JSQL', 0, 'utf8');
+    sb.writeUInt16LE(this.tables.size, 4);
+    let off = 6;
+    for (const [name, t] of this.tables) {
+      const nB = Buffer.from(name, 'utf8');
+      sb[off] = nB.length; off += 1;
+      nB.copy(sb, off); off += nB.length;
+      const sStr = JSON.stringify(t.schema);
+      sb.writeUInt32LE(sStr.length, off); off += 4;
+      sb.writeUInt32LE(t.rowCount, off); off += 4;
+      sb.writeUInt32LE(t.pkBlock||0, off); off += 4;
+      sb.writeUInt32LE(t.dataBlock||0, off); off += 4;
+      sb.writeUInt16LE(t.tableId || 0, off); off += 2;
+      Buffer.from(sStr, 'utf8').copy(sb, off); off += sStr.length;
+    }
+    fs.writeSync(this.fd, sb, 0, SZ.SUPER, 0);
+    fs.fsyncSync(this.fd);
+  }
+
+  _reset() {
+    fs.ftruncateSync(this.fd, SZ.SUPER);
+    const sb = Buffer.alloc(SZ.SUPER); sb.write('JSQL', 0, 'utf8');
+    fs.writeSync(this.fd, sb, 0, SZ.SUPER, 0);
+    this.nBlocks = 1;
+    this._nextTableId = 1;
+    this.tables.clear();
+    this.pkIdx.clear();
+  }
+
+  saveAll(allTables) {
+    this._open();
+    this._reset();
+
+    for (const [name, info] of Object.entries(allTables)) {
+      const { schema, rows, pkField } = info;
+      if (!rows || rows.length === 0) continue;
+
+      const tableId = this._nextTableId++;
+      this.tables.set(name, { schema, rowCount:0, pkBlock:0, dataBlock:0, tableId });
+
+      const pkEntries = [];
+      const rawBlocks = [];
+      const blockIds = [];
+      let curBlock = this._alloc();
+      let buf = Buffer.alloc(SZ.BLOCK);
+      let bp = SZ.HEADER;
+      mkHeader(BT.DATA, curBlock, tableId).copy(buf);
+
+      // Pre-build field encoders once per table
+      const encoders = buildEncoders(schema);
+      // Scratch buffer for row encoding (max row size fits in one block)
+      const scratch = Buffer.allocUnsafe(SZ.DATA);
+
+      for (const row of rows) {
+        const end = encRowInPlace(row, encoders, scratch, 0);
+        if (bp + end > SZ.BLOCK) {
+          const raw = Buffer.from(buf.subarray(SZ.HEADER, bp));
+          rawBlocks.push(raw);
+          blockIds.push(curBlock);
+          buf.writeUInt32LE(bp - SZ.HEADER, 40);
+
+          curBlock = this._alloc();
+          buf = Buffer.alloc(SZ.BLOCK);
+          bp = SZ.HEADER;
+          mkHeader(BT.DATA, curBlock, tableId).copy(buf);
+        }
+
+        const startOff = bp;
+        scratch.copy(buf, bp, 0, end);
+        bp += end;
+
+        if (pkField && row[pkField] !== undefined) {
+          pkEntries.push({ hash: hashPK(row[pkField]), block: curBlock, off: startOff - SZ.HEADER });
+        }
+      }
+
+      if (bp > SZ.HEADER) {
+        const raw = Buffer.from(buf.subarray(SZ.HEADER, bp));
+        rawBlocks.push(raw);
+        blockIds.push(curBlock);
+        buf.writeUInt32LE(bp - SZ.HEADER, 40);
+      }
+
+      // Compress and write blocks
+      for (let i = 0; i < rawBlocks.length; i++) {
+        const bid = blockIds[i];
+        const compBuf = zlib.gzipSync(rawBlocks[i], { level: 1 });
+        const block = Buffer.alloc(SZ.BLOCK);
+        mkHeader(BT.DATA, bid, tableId).copy(block);
+        block.writeUInt32LE(rawBlocks[i].length, 40);
+        block[48] = 1;
+        block.writeUInt32LE(compBuf.length, 44);
+        compBuf.copy(block, SZ.HEADER);
+        finalize(block);
+        fs.writeSync(this.fd, block, 0, SZ.BLOCK, bid * SZ.BLOCK);
+      }
+
+      const t = this.tables.get(name);
+      t.rowCount = rows.length;
+      t.dataBlock = blockIds.length > 0 ? blockIds[blockIds.length - 1] : 0;
+
+      pkEntries.sort((a, b) => a.hash - b.hash);
+      this.pkIdx.set(name, pkEntries);
+      this._saveIdx(name);
+    }
+
+    this._saveSuper();
+    this.dirty = false;
+  }
+
+  writeTable(name, schema, rows, pkField) {
+    this.saveAll({ [name]: { schema, rows, pkField } });
+  }
+
+  readTableSync(name) {
+    this._open();
+    const t = this.tables.get(name);
+    if (!t) return { schema: null, rows: [] };
+
+    const rows = [];
+    for (let bid = 1; bid < this.nBlocks; bid++) {
+      const buf = Buffer.alloc(SZ.BLOCK);
+      try { fs.readSync(this.fd, buf, 0, SZ.BLOCK, bid * SZ.BLOCK); } catch(e) { continue; }
+      if (buf[36] !== BT.DATA || buf[37] !== t.tableId) continue;
+      if (!verifyHash(buf)) continue;
+
+      const compFlag = buf[48];
+      const raw = compFlag
+        ? zlib.gunzipSync(buf.slice(SZ.HEADER, SZ.HEADER + buf.readUInt32LE(44)))
+        : buf.slice(SZ.HEADER, SZ.HEADER + buf.readUInt32LE(40));
+
+      let off = 0;
+      while (off + 4 <= raw.length) {
+        const l = raw.readUInt32LE(off);
+        if (l === 0 || off + 4 + l > raw.length) break;
+        const r = decRow(raw, off);
+        rows.push(r.row);
+        off = r.off;
+      }
+    }
+    return { schema: t.schema, rows };
+  }
+
+  readTable(name) {
+    return this.readTableSync(name);
+  }
+
+  findRow(name, pkValue) {
+    const entries = this.pkIdx.get(name);
+    if (!entries || entries.length === 0) return null;
+    const h = hashPK(pkValue);
+    let lo = 0, hi = entries.length - 1;
+    while (lo <= hi) {
+      const m = (lo + hi) >>> 1;
+      const e = entries[m];
+      if (e.hash === h) return this._readInBlock(e.block, e.off);
+      if (e.hash < h) lo = m + 1;
+      else hi = m - 1;
+    }
+    return null;
+  }
+
+  _readInBlock(blockId, off) {
+    const buf = Buffer.alloc(SZ.BLOCK);
+    fs.readSync(this.fd, buf, 0, SZ.BLOCK, blockId * SZ.BLOCK);
+    const compFlag = buf[48];
+    const compSz = buf.readUInt32LE(44);
+    const dataSz = buf.readUInt32LE(40);
+    let raw;
+    if (compFlag) {
+      raw = zlib.gunzipSync(buf.slice(SZ.HEADER, SZ.HEADER + compSz));
+    } else {
+      raw = buf.slice(SZ.HEADER, SZ.HEADER + dataSz);
+    }
+    const r = decRow(raw, off);
+    return r.row;
+  }
+
+  appendRows(name, schema, rows, pkField) {
+    this._open();
+    const t = this.tables.get(name);
+    if (!t) { this.writeTable(name, schema, rows, pkField); return; }
+    t.schema = schema;
+
+    const { rows: oldRows } = this.readTable(name);
+    oldRows.push(...rows);
+    this.writeTable(name, schema, oldRows, pkField);
+  }
+}
+
+module.exports = JSQLFormat;

package/lib/jsql_format2.js

@@ -0,0 +1,448 @@
+const zlib = require('zlib');
+const crypto = require('crypto');
+const fs = require('fs');
+
+const MAGIC = Buffer.from('4A534F4C', 'hex');
+const VERSION = 1;
+const BLOCK_SIZE = 65536;
+const BLOCK_HEADER_SIZE = 64;
+const BLOCK_DATA_SIZE = BLOCK_SIZE - BLOCK_HEADER_SIZE;
+const SUPERBLOCK_SIZE = 4096;
+
+const BT = { FREE: 0, DATA: 1, PK_INDEX: 2, SUPER: 3 };
+
+const TYPE = {
+  NULL: 0, BOOL: 1, INT: 2, UINT: 3, FLOAT: 4,
+  STRING: 5, BUFFER: 6, ARRAY: 7, OBJECT: 8
+};
+
+function typeTag(v) {
+  if (v === null || v === undefined) return TYPE.NULL;
+  if (typeof v === 'boolean') return TYPE.BOOL;
+  if (typeof v === 'number') return Number.isInteger(v) ? TYPE.INT : TYPE.FLOAT;
+  if (typeof v === 'string') return TYPE.STRING;
+  if (v instanceof Buffer) return TYPE.BUFFER;
+  return Array.isArray(v) ? TYPE.ARRAY : TYPE.OBJECT;
+}
+
+function encodeValue(v) {
+  const tag = typeTag(v);
+  switch (tag) {
+    case TYPE.NULL: return Buffer.from([tag]);
+    case TYPE.BOOL: return Buffer.from([tag, v ? 1 : 0]);
+    case TYPE.INT: { const b = Buffer.alloc(5); b[0] = tag; b.writeInt32LE(v, 1); return b; }
+    case TYPE.UINT: { const b = Buffer.alloc(5); b[0] = tag; b.writeUInt32LE(v, 1); return b; }
+    case TYPE.FLOAT: { const b = Buffer.alloc(9); b[0] = tag; b.writeDoubleLE(v, 1); return b; }
+    case TYPE.STRING: { const s = Buffer.from(v, 'utf8'); const b = Buffer.alloc(5 + s.length); b[0] = tag; b.writeUInt32LE(s.length, 1); s.copy(b, 5); return b; }
+    case TYPE.BUFFER: { const b = Buffer.alloc(5 + v.length); b[0] = tag; b.writeUInt32LE(v.length, 1); v.copy(b, 5); return b; }
+    case TYPE.ARRAY: {
+      const parts = [Buffer.from([tag]), encodeValue(v.length)];
+      for (const item of v) parts.push(encodeValue(item));
+      return Buffer.concat(parts);
+    }
+    case TYPE.OBJECT: {
+      const keys = Object.keys(v);
+      const parts = [Buffer.from([tag]), encodeValue(keys.length)];
+      for (const k of keys) { parts.push(encodeValue(k)); parts.push(encodeValue(v[k])); }
+      return Buffer.concat(parts);
+    }
+  }
+}
+
+function decodeValue(buf, off) {
+  const tag = buf[off++];
+  switch (tag) {
+    case TYPE.NULL: return { v: null, off };
+    case TYPE.BOOL: return { v: buf[off] !== 0, off: off + 1 };
+    case TYPE.INT: return { v: buf.readInt32LE(off), off: off + 4 };
+    case TYPE.UINT: return { v: buf.readUInt32LE(off), off: off + 4 };
+    case TYPE.FLOAT: return { v: buf.readDoubleLE(off), off: off + 8 };
+    case TYPE.STRING: { const len = buf.readUInt32LE(off); off += 4; return { v: buf.toString('utf8', off, off + len), off: off + len }; }
+    case TYPE.BUFFER: { const len = buf.readUInt32LE(off); off += 4; return { v: Buffer.from(buf.subarray(off, off + len)), off: off + len }; }
+    case TYPE.ARRAY: {
+      let r = decodeValue(buf, off); off = r.off;
+      const len = r.v, arr = new Array(len);
+      for (let i = 0; i < len; i++) { r = decodeValue(buf, off); arr[i] = r.v; off = r.off; }
+      return { v: arr, off };
+    }
+    case TYPE.OBJECT: {
+      let r = decodeValue(buf, off); off = r.off;
+      const len = r.v, obj = {};
+      for (let i = 0; i < len; i++) { r = decodeValue(buf, off); const k = r.v; off = r.off; r = decodeValue(buf, off); obj[k] = r.v; off = r.off; }
+      return { v: obj, off };
+    }
+    default: throw new Error('Unknown type: ' + tag);
+  }
+}
+
+function makeBlockHeader(type, seq) {
+  const h = Buffer.alloc(BLOCK_HEADER_SIZE);
+  h.writeUInt32LE(seq, 32);
+  h[36] = type;
+  return h;
+}
+
+function finalizeBlock(block) {
+  const hash = crypto.createHash('sha256').update(block.subarray(32)).digest();
+  hash.copy(block, 0);
+  return block;
+}
+
+function verifyBlockHash(block) {
+  const stored = block.slice(0, 32);
+  const computed = crypto.createHash('sha256').update(block.subarray(32)).digest();
+  return stored.equals(computed);
+}
+
+function encodeRow(row, schema) {
+  const fields = Object.keys(schema).filter(f => f !== '_softDelete');
+  const parts = [encodeValue(fields.length)];
+  for (const f of fields) {
+    parts.push(encodeValue(f));
+    parts.push(encodeValue(row[f]));
+  }
+  return Buffer.concat(parts);
+}
+
+function decodeRow(buf, off) {
+  let r = decodeValue(buf, off); off = r.off;
+  const fieldCount = r.v;
+  const row = {};
+  for (let i = 0; i < fieldCount; i++) {
+    r = decodeValue(buf, off); const field = r.v; off = r.off;
+    r = decodeValue(buf, off); row[field] = r.v; off = r.off;
+  }
+  return { row, off };
+}
+
+class JSQLFile {
+  constructor(path, blockSize = BLOCK_SIZE) {
+    this.path = path;
+    this.blockSize = blockSize;
+    this.dataSize = blockSize - BLOCK_HEADER_SIZE;
+    this.fd = null;
+    this.tables = new Map();
+    this.pkIndex = new Map();
+    this.totalBlocks = 1;
+    this.freeBlocks = [];
+    this.dirty = false;
+  }
+
+  _rowSize(row, schema) {
+    return encodeRow(row, schema).length + 4;
+  }
+
+  open() {
+    if (this.fd) return;
+    if (fs.existsSync(this.path)) {
+      this.fd = fs.openSync(this.path, 'r+');
+      this._load();
+    } else {
+      this.fd = fs.openSync(this.path, 'w+');
+      this._initFile();
+    }
+  }
+
+  close() {
+    if (!this.fd) return;
+    if (this.dirty) this._flush();
+    fs.closeSync(this.fd);
+    this.fd = null;
+  }
+
+  _initFile() {
+    fs.ftruncateSync(this.fd, SUPERBLOCK_SIZE);
+  }
+
+  _load() {
+    const sb = Buffer.alloc(SUPERBLOCK_SIZE);
+    fs.readSync(this.fd, sb, 0, SUPERBLOCK_SIZE, 0);
+    const magic = sb.slice(0, 4);
+    if (magic.compare(MAGIC) !== 0) throw new Error('Not a JSQL file');
+    const version = sb.readUInt16LE(4);
+    this.blockSize = sb.readUInt32LE(6) || BLOCK_SIZE;
+    this.dataSize = this.blockSize - BLOCK_HEADER_SIZE;
+
+    const tableCount = sb.readUInt16LE(10);
+    let off = 12;
+    for (let i = 0; i < tableCount; i++) {
+      const nameLen = Math.min(sb[off], 64);
+      const name = sb.toString('utf8', off + 1, off + 1 + nameLen);
+      off += 1 + nameLen;
+      const schemaLen = sb.readUInt32LE(off); off += 4;
+      const rowCount = sb.readUInt32LE(off); off += 4;
+      const pkBlock = sb.readUInt32LE(off); off += 4;
+      const dataBlock = sb.readUInt32LE(off); off += 4;
+      const schemaStr = sb.toString('utf8', off, off + schemaLen);
+      off += schemaLen;
+
+      this.tables.set(name, {
+        schema: JSON.parse(schemaStr),
+        rowCount, pkBlock, dataBlock
+      });
+
+      // Load PK index
+      this.pkIndex.set(name, this._loadPKIndex(pkBlock));
+    }
+
+    this.totalBlocks = Math.max(1, Math.ceil(fs.fstatSync(this.fd).size / this.blockSize));
+  }
+
+  _loadPKIndex(blockId) {
+    const entries = [];
+    if (blockId < 1 || blockId >= this.totalBlocks) return entries;
+    const buf = Buffer.alloc(this.blockSize);
+    fs.readSync(this.fd, buf, 0, this.blockSize, blockId * this.blockSize);
+    if (!verifyBlockHash(buf)) throw new Error('PK index block corrupted');
+
+    const type = buf[36];
+    if (type !== BT.PK_INDEX) throw new Error('Not a PK index block');
+
+    let off = BLOCK_HEADER_SIZE;
+    const count = buf.readUInt32LE(off); off += 4;
+    for (let i = 0; i < count; i++) {
+      const pkHash = buf.readBigUInt64LE(off); off += 8;
+      const pkOff = buf.readUInt32LE(off); off += 4;
+      const rowLen = buf.readUInt32LE(off); off += 4;
+      const dataBlock = buf.readUInt32LE(off); off += 4;
+      const dataOff = buf.readUInt32LE(off); off += 4;
+      entries.push({ pkHash, pkOff, rowLen, dataBlock, dataOff });
+    }
+    return entries;
+  }
+
+  _savePKIndex(name) {
+    const entries = this.pkIndex.get(name) || [];
+    const table = this.tables.get(name);
+    const buf = Buffer.alloc(this.blockSize);
+    makeBlockHeader(BT.PK_INDEX, table ? table.pkBlock : 0).copy(buf);
+
+    let off = BLOCK_HEADER_SIZE;
+    buf.writeUInt32LE(entries.length, off); off += 4;
+    for (const e of entries) {
+      buf.writeBigUInt64LE(e.pkHash, off); off += 8;
+      buf.writeUInt32LE(e.pkOff, off); off += 4;
+      buf.writeUInt32LE(e.rowLen, off); off += 4;
+      buf.writeUInt32LE(e.dataBlock, off); off += 4;
+      buf.writeUInt32LE(e.dataOff, off); off += 4;
+    }
+    finalizeBlock(buf);
+
+    const blockId = table ? table.pkBlock : this._allocBlock();
+    fs.writeSync(this.fd, buf, 0, this.blockSize, blockId * this.blockSize);
+    if (table) table.pkBlock = blockId;
+    return blockId;
+  }
+
+  _allocBlock() {
+    if (this.freeBlocks.length) return this.freeBlocks.pop();
+    return this.totalBlocks++;
+  }
+
+  _flush() {
+    // Save all PK indexes and superblock
+    for (const name of this.tables.keys()) {
+      this._savePKIndex(name);
+    }
+    this._saveSuperBlock();
+    this.dirty = false;
+  }
+
+  _saveSuperBlock() {
+    const sb = Buffer.alloc(SUPERBLOCK_SIZE);
+    MAGIC.copy(sb, 0);
+    sb.writeUInt16LE(VERSION, 4);
+    sb.writeUInt32LE(this.blockSize, 6);
+    sb.writeUInt16LE(this.tables.size, 10);
+
+    let off = 12;
+    for (const [name, table] of this.tables) {
+      const nameBuf = Buffer.from(name, 'utf8');
+      sb[off] = nameBuf.length; off += 1;
+      nameBuf.copy(sb, off); off += nameBuf.length;
+      const schemaStr = JSON.stringify(table.schema);
+      sb.writeUInt32LE(schemaStr.length, off); off += 4;
+      sb.writeUInt32LE(table.rowCount, off); off += 4;
+      sb.writeUInt32LE(table.pkBlock || 0, off); off += 4;
+      sb.writeUInt32LE(table.dataBlock || 0, off); off += 4;
+      Buffer.from(schemaStr, 'utf8').copy(sb, off); off += schemaStr.length;
+    }
+
+    fs.writeSync(this.fd, sb, 0, SUPERBLOCK_SIZE, 0);
+    fs.fsyncSync(this.fd);
+  }
+
+  writeTable(name, schema, rows, pkField) {
+    this.open();
+    const table = this.tables.get(name) || { schema, rowCount: 0, pkBlock: 0, dataBlock: 0 };
+    table.schema = schema;
+    this.tables.set(name, table);
+
+    const pkEntries = [];
+    let currentBlock = this._allocBlock();
+    let blockBuf = Buffer.alloc(this.blockSize);
+    let blockOff = BLOCK_HEADER_SIZE;
+    let seq = currentBlock;
+    let hasData = false;
+
+    makeBlockHeader(BT.DATA, seq).copy(blockBuf);
+
+    for (const row of rows) {
+      const rowData = encodeRow(row, schema);
+      const rowLen = rowData.length + 4;
+      const lenBuf = Buffer.alloc(4);
+      lenBuf.writeUInt32LE(rowData.length);
+
+      if (blockOff + rowLen > this.blockSize) {
+        // Flush current block
+        blockBuf.writeUInt16LE(0, 38); // rowCount
+        // Actually write row count
+        // We need to track rows per block properly
+        const written = blockOff - BLOCK_HEADER_SIZE;
+        blockBuf.writeUInt32LE(written, 40); // data size
+
+        // Compress
+        const rawData = blockBuf.slice(BLOCK_HEADER_SIZE, blockOff);
+        const compressed = zlib.gzipSync(rawData, { level: 6 });
+        compressed.copy(blockBuf, BLOCK_HEADER_SIZE);
+        blockBuf.writeUInt32LE(compressed.length, 44);
+        blockBuf[48] = 1; // compressed flag
+
+        // TODO: track row count properly
+        finalizeBlock(blockBuf);
+        fs.writeSync(this.fd, blockBuf, 0, this.blockSize, currentBlock * this.blockSize);
+
+        // New block
+        currentBlock = this._allocBlock();
+        seq = currentBlock;
+        blockBuf = Buffer.alloc(this.blockSize);
+        blockOff = BLOCK_HEADER_SIZE;
+        makeBlockHeader(BT.DATA, seq).copy(blockBuf);
+      }
+
+      lenBuf.copy(blockBuf, blockOff); blockOff += 4;
+      rowData.copy(blockBuf, blockOff); blockOff += rowData.length;
+      hasData = true;
+
+      if (pkField && row[pkField] !== undefined) {
+        const pkVal = row[pkField];
+        const pkHash = BigInt(hashPK(pkVal));
+        pkEntries.push({ pkHash, pkOff: 0, rowLen: rowData.length, dataBlock: currentBlock, dataOff: blockOff - rowData.length - 4, pkVal });
+      }
+    }
+
+    if (hasData) {
+      const written = blockOff - BLOCK_HEADER_SIZE;
+      blockBuf.writeUInt32LE(written, 40);
+      const rawData = blockBuf.slice(BLOCK_HEADER_SIZE, blockOff);
+      const compressed = zlib.gzipSync(rawData, { level: 6 });
+      compressed.copy(blockBuf, BLOCK_HEADER_SIZE);
+      blockBuf.writeUInt32LE(compressed.length, 44);
+      blockBuf[48] = 1;
+      finalizeBlock(blockBuf);
+      fs.writeSync(this.fd, blockBuf, 0, this.blockSize, currentBlock * this.blockSize);
+    }
+
+    table.rowCount = rows.length;
+    table.dataBlock = seq;
+
+    // Sort PK entries by hash for binary search
+    pkEntries.sort((a, b) => a.pkHash < b.pkHash ? -1 : a.pkHash > b.pkHash ? 1 : 0);
+    this.pkIndex.set(name, pkEntries.map(e => ({ pkHash: e.pkHash, pkOff: 0, rowLen: e.rowLen, dataBlock: e.dataBlock, dataOff: e.dataOff })));
+    this._savePKIndex(name);
+    this._saveSuperBlock();
+    this.dirty = false;
+  }
+
+  readTable(name) {
+    this.open();
+    const table = this.tables.get(name);
+    if (!table) return { schema: null, rows: [] };
+
+    const rows = [];
+    const entries = this.pkIndex.get(name) || [];
+
+    // Collect unique blocks
+    const blockMap = new Map();
+    for (const e of entries) {
+      if (!blockMap.has(e.dataBlock)) blockMap.set(e.dataBlock, new Map());
+      blockMap.get(e.dataBlock).set(e.dataOff, e);
+    }
+
+    for (const [blockId, blockEntries] of blockMap) {
+      const buf = Buffer.alloc(this.blockSize);
+      fs.readSync(this.fd, buf, 0, this.blockSize, blockId * this.blockSize);
+      if (!verifyBlockHash(buf)) throw new Error('Block corrupted: ' + blockId);
+
+      const compressed = buf[48] === 1;
+      const compSize = buf.readUInt32LE(44);
+      let rawData;
+      if (compressed) {
+        rawData = zlib.gunzipSync(buf.slice(BLOCK_HEADER_SIZE, BLOCK_HEADER_SIZE + compSize));
+      } else {
+        rawData = buf.slice(BLOCK_HEADER_SIZE, BLOCK_HEADER_SIZE + buf.readUInt32LE(40));
+      }
+
+      // Read all rows from this block
+      let off = 0;
+      while (off < rawData.length) {
+        const rowLen = rawData.readUInt32LE(off); off += 4;
+        if (rowLen === 0 || off + rowLen > rawData.length) break;
+        const r = decodeRow(rawData, off);
+        rows.push(r.row);
+        off += rowLen;
+      }
+    }
+
+    return { schema: table.schema, rows };
+  }
+
+  readRowByName(name, pkField, pkValue) {
+    this.open();
+    const table = this.tables.get(name);
+    if (!table) return null;
+
+    const entries = this.pkIndex.get(name) || [];
+    const targetHash = BigInt(hashPK(pkValue));
+
+    // Binary search
+    let lo = 0, hi = entries.length - 1;
+    while (lo <= hi) {
+      const mid = (lo + hi) >>> 1;
+      const e = entries[mid];
+      if (e.pkHash === targetHash) return this._readRowAt(e.dataBlock, e.dataOff, e.rowLen);
+      if (e.pkHash < targetHash) lo = mid + 1;
+      else hi = mid - 1;
+    }
+    return null;
+  }
+
+  _readRowAt(blockId, dataOff, rowLen) {
+    const buf = Buffer.alloc(this.blockSize);
+    fs.readSync(this.fd, buf, 0, this.blockSize, blockId * this.blockSize);
+
+    const compressed = buf[48] === 1;
+    const compSize = buf.readUInt32LE(44);
+    let rawData;
+    if (compressed) {
+      rawData = zlib.gunzipSync(buf.slice(BLOCK_HEADER_SIZE, BLOCK_HEADER_SIZE + compSize));
+    } else {
+      rawData = buf.slice(BLOCK_HEADER_SIZE, BLOCK_HEADER_SIZE + buf.readUInt32LE(40));
+    }
+
+    const off = dataOff - (blockId * this.blockSize + BLOCK_HEADER_SIZE) + BLOCK_HEADER_SIZE;
+    const r = decodeRow(rawData, dataOff);
+    return r.row;
+  }
+}
+
+function hashPK(val) {
+  const s = String(val);
+  let h = 0;
+  for (let i = 0; i < s.length; i++) {
+    h = (Math.imul(h, 31) + s.charCodeAt(i)) | 0;
+  }
+  return h >>> 0;
+}
+
+module.exports = { JSQLFile, hashPK, encodeRow, decodeRow };

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "jsql-neo",
-    "version": "2.2.0",
+    "version": "2.2.1",
     "description": "JSQL-NEO — Pure JavaScript embedded database with B-Tree indexes, WAL, hash JOIN, Redis-compatible Cache, and MySQL-style error codes",
     "main": "index.js",
     "files": [