jsql-neo 2.1.4 → 2.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/index.js CHANGED
@@ -15,5 +15,6 @@ module.exports = {
15
15
  BTree: require('./lib/btree'),
16
16
  Cache: require('./lib/cache'),
17
17
  JSQL_Error: require('./lib/errors').JSQL_Error,
18
- ErrorCodes: require('./lib/errors').ErrorCodes
18
+ ErrorCodes: require('./lib/errors').ErrorCodes,
19
+ JSQLFormat: require('./lib/jsql_format')
19
20
  };
@@ -0,0 +1,122 @@
1
+ const { Worker, MessageChannel, receiveMessageOnPort } = require('worker_threads');
2
+ const path = require('path');
3
+ const os = require('os');
4
+ const zlib = require('zlib');
5
+
6
+ const POOL_SIZE = Math.max(2, Math.min(os.cpus().length - 1, 8));
7
+
8
+ let pool = null;
9
+
10
+ class SyncCompressionPool {
11
+ constructor(size) {
12
+ this.size = size;
13
+ this.workers = [];
14
+ this.ports = [];
15
+ this.nextId = 1;
16
+ const wp = path.join(__dirname, 'compress_worker.js');
17
+
18
+ for (let i = 0; i < size; i++) {
19
+ const { port1, port2 } = new MessageChannel();
20
+ const worker = new Worker(wp);
21
+ worker.postMessage({ type: 'init', port: port2 }, [port2]);
22
+ worker.unref();
23
+ this.workers.push({ worker, port: port1 });
24
+ this.ports.push(port1);
25
+ }
26
+ }
27
+
28
+ _getIdle() {
29
+ for (const p of this.ports) {
30
+ if (!p._busy) return p;
31
+ }
32
+ return null;
33
+ }
34
+
35
+ _copyBuffer(data) {
36
+ const src = Buffer.isBuffer(data) ? data : Buffer.from(data);
37
+ const ab = new ArrayBuffer(src.length);
38
+ const dst = Buffer.from(ab);
39
+ src.copy(dst);
40
+ return dst;
41
+ }
42
+
43
+ _sendMsg(port, msg, data) {
44
+ const copy = this._copyBuffer(data);
45
+ port._busy = true;
46
+ port.postMessage({ ...msg, data: copy.buffer, level: msg.level || 1 }, [copy.buffer]);
47
+ }
48
+
49
+ _waitResult(port) {
50
+ let r = null;
51
+ while (!r) { r = receiveMessageOnPort(port); }
52
+ port._busy = false;
53
+ if (r.message.error) throw new Error(r.message.error);
54
+ return r.message;
55
+ }
56
+
57
+ compressSync(data, level = 1) {
58
+ const port = this._getIdle();
59
+ if (!port) {
60
+ return { result: zlib.gzipSync(Buffer.isBuffer(data) ? data : Buffer.from(data), { level }).buffer };
61
+ }
62
+ const id = this.nextId++;
63
+ this._sendMsg(port, { type: 'compress', id, level }, data);
64
+ return this._waitResult(port);
65
+ }
66
+
67
+ decompressSync(data) {
68
+ const port = this._getIdle();
69
+ if (!port) {
70
+ return { result: zlib.gunzipSync(Buffer.isBuffer(data) ? data : Buffer.from(data)).buffer };
71
+ }
72
+ const id = this.nextId++;
73
+ this._sendMsg(port, { type: 'decompress', id }, data);
74
+ return this._waitResult(port);
75
+ }
76
+
77
+ compressBatchSync(blocks, level = 1) {
78
+ const n = blocks.length;
79
+ const results = new Array(n);
80
+ let submitted = 0;
81
+ let completed = 0;
82
+
83
+ const trySubmit = () => {
84
+ while (submitted < n) {
85
+ const port = this._getIdle();
86
+ if (!port) break;
87
+ const idx = submitted++;
88
+ port._idx = idx;
89
+ port._busy = true;
90
+ const copy = this._copyBuffer(blocks[idx]);
91
+ port.postMessage({ type: 'compress', id: this.nextId++, data: copy.buffer, level }, [copy.buffer]);
92
+ }
93
+ };
94
+
95
+ trySubmit();
96
+ while (completed < n) {
97
+ for (const port of this.ports) {
98
+ if (!port._busy) continue;
99
+ const msg = receiveMessageOnPort(port);
100
+ if (msg) {
101
+ results[port._idx] = msg.message;
102
+ port._busy = false;
103
+ completed++;
104
+ trySubmit();
105
+ }
106
+ }
107
+ }
108
+ return results;
109
+ }
110
+
111
+ terminate() {
112
+ for (const { worker } of this.workers) worker.terminate();
113
+ pool = null;
114
+ }
115
+ }
116
+
117
+ function getPool() {
118
+ if (!pool) pool = new SyncCompressionPool(POOL_SIZE);
119
+ return pool;
120
+ }
121
+
122
+ module.exports = { getPool, SyncCompressionPool };
@@ -0,0 +1,32 @@
1
+ const { parentPort } = require('worker_threads');
2
+ const zlib = require('zlib');
3
+
4
+ let commPort = null;
5
+
6
+ parentPort.on('message', (msg) => {
7
+ if (msg.type === 'init' && msg.port) {
8
+ commPort = msg.port;
9
+ commPort.on('message', (task) => {
10
+ try {
11
+ if (task.type === 'compress') {
12
+ const buf = Buffer.from(task.data);
13
+ const result = zlib.gzipSync(buf, { level: task.level || 1 });
14
+ const ab = new ArrayBuffer(result.length);
15
+ const view = new Uint8Array(ab);
16
+ for (let i = 0; i < result.length; i++) view[i] = result[i];
17
+ commPort.postMessage({ id: task.id, result: ab, len: result.length }, [ab]);
18
+ } else if (task.type === 'decompress') {
19
+ const buf = Buffer.from(task.data);
20
+ const result = zlib.gunzipSync(buf);
21
+ const ab = new ArrayBuffer(result.length);
22
+ const view = new Uint8Array(ab);
23
+ for (let i = 0; i < result.length; i++) view[i] = result[i];
24
+ commPort.postMessage({ id: task.id, result: ab, len: result.length }, [ab]);
25
+ }
26
+ } catch (e) {
27
+ commPort.postMessage({ id: task.id, error: e.message });
28
+ }
29
+ });
30
+ commPort.unref();
31
+ }
32
+ });
package/lib/database.js CHANGED
@@ -9,6 +9,7 @@ const path = require('path');
9
9
  const crypto = require('crypto');
10
10
  const Table = require('./table');
11
11
  const { createError } = require('./errors');
12
+ const JSQLFormat = require('./jsql_format');
12
13
 
13
14
  class Database {
14
15
  /**
@@ -57,6 +58,10 @@ class Database {
57
58
  // 事务隔离
58
59
  this._isolationLevel = options.isolationLevel || 'READ_COMMITTED';
59
60
 
61
+ // JSql 格式 (binary block-based)
62
+ this._jsqlMode = this._filePath && this._filePath.endsWith('.jsql');
63
+ this._jsqlFormat = this._jsqlMode ? new JSQLFormat(this._filePath) : null;
64
+
60
65
  // 文件锁
61
66
  this._fileLockEnabled = options.fileLock === true && !this._memoryMode;
62
67
  this._lockFd = null;
@@ -898,6 +903,20 @@ class Database {
898
903
 
899
904
  if (this._memoryMode) return;
900
905
 
906
+ if (this._jsqlMode) {
907
+ const all = {};
908
+ for (const [name, table] of Object.entries(this._tables)) {
909
+ if (name === '__meta__') continue;
910
+ const pkField = Object.keys(table._schema).find(f => table._schema[f].primaryKey);
911
+ all[name] = { schema: table._schema, rows: table._rows, pkField };
912
+ }
913
+ this._jsqlFormat.saveAll(all);
914
+ this._jsqlFormat._close();
915
+ this._dirty = false;
916
+ this._checkpoint();
917
+ return;
918
+ }
919
+
901
920
  const data = { __schema__: {}, __meta__: { version: this._getCurrentVersion() } };
902
921
  for (const [name, table] of Object.entries(this._tables)) {
903
922
  if (name === '__meta__') continue;
@@ -917,6 +936,18 @@ class Database {
917
936
 
918
937
  _load() {
919
938
  try {
939
+ if (this._jsqlMode) {
940
+ this._jsqlFormat._open();
941
+ for (const [name, t] of this._jsqlFormat.tables) {
942
+ if (!this._tables[name]) this.createTable(name, t.schema);
943
+ if (t.rowCount > 0) {
944
+ const { rows } = this._jsqlFormat.readTableSync(name);
945
+ this._tables[name]._loadRows(rows);
946
+ }
947
+ }
948
+ return;
949
+ }
950
+
920
951
  const json = this._readFile(this._filePath);
921
952
  const data = JSON.parse(json);
922
953
 
@@ -964,6 +995,7 @@ class Database {
964
995
  if (this._dirty && !this._memoryMode) {
965
996
  this.save();
966
997
  }
998
+ if (this._jsqlFormat) this._jsqlFormat._close();
967
999
  this._releaseLock();
968
1000
  this._tables = {};
969
1001
  }
@@ -0,0 +1,478 @@
1
+ const zlib = require('zlib');
2
+ const crypto = require('crypto');
3
+ const fs = require('fs');
4
+
5
+
6
+ const SZ = {
7
+ SUPER: 4096,
8
+ HEADER: 64,
9
+ BLOCK: 4096,
10
+ DATA: 4096 - 64
11
+ };
12
+
13
+ const BT = { FREE: 0, DATA: 1, IDX: 2 };
14
+
15
+ const TYPE = { NUL:0, BOL:1, I32:2, F64:3, STR:4, BUF:5, ARR:6, OBJ:7 };
16
+
17
+ function tag(v) {
18
+ if (v===null||v===undefined) return TYPE.NUL;
19
+ if (typeof v==='boolean') return TYPE.BOL;
20
+ if (typeof v==='number') return Number.isInteger(v)?TYPE.I32:TYPE.F64;
21
+ if (typeof v==='string') return TYPE.STR;
22
+ if (v instanceof Buffer) return TYPE.BUF;
23
+ return Array.isArray(v)?TYPE.ARR:TYPE.OBJ;
24
+ }
25
+
26
+ function enc(v) {
27
+ const t = tag(v);
28
+ switch (t) {
29
+ case TYPE.NUL: return Buffer.from([t]);
30
+ case TYPE.BOL: return Buffer.from([t, v?1:0]);
31
+ case TYPE.I32: { const b=Buffer.alloc(5); b[0]=t; b.writeInt32LE(v,1); return b; }
32
+ case TYPE.F64: { const b=Buffer.alloc(9); b[0]=t; b.writeDoubleLE(v,1); return b; }
33
+ case TYPE.STR: { const s=Buffer.from(v,'utf8'); const b=Buffer.alloc(5+s.length); b[0]=t; b.writeUInt32LE(s.length,1); s.copy(b,5); return b; }
34
+ case TYPE.BUF: { const b=Buffer.alloc(5+v.length); b[0]=t; b.writeUInt32LE(v.length,1); v.copy(b,5); return b; }
35
+ case TYPE.ARR: { const p=[Buffer.from([t]),enc(v.length)]; for(const i of v) p.push(enc(i)); return Buffer.concat(p); }
36
+ case TYPE.OBJ: { const k=Object.keys(v); const p=[Buffer.from([t]),enc(k.length)]; for(const f of k){p.push(enc(f));p.push(enc(v[f]));} return Buffer.concat(p); }
37
+ }
38
+ }
39
+
40
+ function dec(buf, off) {
41
+ const t = buf[off++];
42
+ switch (t) {
43
+ case TYPE.NUL: return {v:null,off};
44
+ case TYPE.BOL: return {v:buf[off]!==0,off:off+1};
45
+ case TYPE.I32: return {v:buf.readInt32LE(off),off:off+4};
46
+ case TYPE.F64: return {v:buf.readDoubleLE(off),off:off+8};
47
+ case TYPE.STR: { const l=buf.readUInt32LE(off);off+=4; return {v:buf.toString('utf8',off,off+l),off:off+l}; }
48
+ case TYPE.BUF: { const l=buf.readUInt32LE(off);off+=4; return {v:Buffer.from(buf.subarray(off,off+l)),off:off+l}; }
49
+ case TYPE.ARR: { let r=dec(buf,off);off=r.off; const l=r.v,a=new Array(l); for(let i=0;i<l;i++){r=dec(buf,off);a[i]=r.v;off=r.off;} return {v:a,off}; }
50
+ case TYPE.OBJ: { let r=dec(buf,off);off=r.off; const l=r.v,o={}; for(let i=0;i<l;i++){r=dec(buf,off);const k=r.v;off=r.off;r=dec(buf,off);o[k]=r.v;off=r.off;} return {v:o,off}; }
51
+ default: throw new Error('bad type '+t);
52
+ }
53
+ }
54
+
55
+ // Pre-encoded field names for faster row encoding
56
+ function buildFieldEncoders(schema) {
57
+ const fields = Object.keys(schema).filter(f => f !== '_softDelete');
58
+ return fields.map(f => enc(f));
59
+ }
60
+
61
+ function encRowFast(row, fieldEncoders) {
62
+ // Estimate: 4(len) + sum of field values
63
+ // fieldEncoders includes pre-encoded field names
64
+ let size = 4;
65
+ for (let i = 0; i < fieldEncoders.length; i++) {
66
+ size += fieldEncoders[i].length;
67
+ const v = row[Object.keys(row)[i] || '']; // fallback
68
+ }
69
+ // Just use the buffer approach
70
+ const chunks = [Buffer.alloc(4)];
71
+ for (let i = 0; i < fieldEncoders.length; i++) {
72
+ chunks.push(fieldEncoders[i], enc(row[Object.keys(row)[i]]));
73
+ }
74
+ const data = Buffer.concat(chunks);
75
+ data.writeUInt32LE(data.length - 4, 0);
76
+ return data;
77
+ }
78
+
79
+ function writeEnc(buf, off, v) {
80
+ if (v === null || v === undefined) { buf[off] = 0; return off + 1; }
81
+ if (typeof v === 'boolean') { buf[off] = 1; buf[off+1] = v ? 1 : 0; return off + 2; }
82
+ if (typeof v === 'number') {
83
+ if (Number.isInteger(v)) { buf[off] = 2; buf.writeInt32LE(v, off+1); return off + 5; }
84
+ buf[off] = 3; buf.writeDoubleLE(v, off+1); return off + 9;
85
+ }
86
+ if (typeof v === 'string') {
87
+ buf[off] = 4;
88
+ const encoded = Buffer.from(v, 'utf8');
89
+ buf.writeUInt32LE(encoded.length, off+1);
90
+ encoded.copy(buf, off+5);
91
+ return off + 5 + encoded.length;
92
+ }
93
+ const e = enc(v);
94
+ e.copy(buf, off);
95
+ return off + e.length;
96
+ }
97
+
98
+ function encRowInPlace(row, encoders, scratch, offStart) {
99
+ let off = offStart + 4;
100
+ for (let i = 0; i < encoders.length; i++) {
101
+ encoders[i].nameEnc.copy(scratch, off); off += encoders[i].nameEnc.length;
102
+ off = writeEnc(scratch, off, row[encoders[i].name]);
103
+ }
104
+ scratch.writeUInt32LE(off - offStart - 4, offStart);
105
+ return off;
106
+ }
107
+
108
+ // Build reusable field encoders from schema
109
+ function buildEncoders(schema) {
110
+ return Object.keys(schema).filter(f => f !== '_softDelete').map(name => ({ name, nameEnc: enc(name) }));
111
+ }
112
+
113
+ const encRow = encRowInPlace;
114
+
115
+ function decRow(buf, off) {
116
+ const l = buf.readUInt32LE(off); off += 4;
117
+ const end = off + l;
118
+ const row = {};
119
+ while (off < end) {
120
+ const r1 = dec(buf, off); const f = r1.v; off = r1.off;
121
+ const r2 = dec(buf, off); row[f] = r2.v; off = r2.off;
122
+ }
123
+ return { row, off };
124
+ }
125
+
126
+ function hashPK(val) {
127
+ let h = 0;
128
+ const s = String(val);
129
+ for (let i = 0; i < s.length; i++) h = (Math.imul(h,31) + s.charCodeAt(i)) | 0;
130
+ return h >>> 0;
131
+ }
132
+
133
+ function mkHeader(type, seq, tableId) {
134
+ const h = Buffer.alloc(SZ.HEADER);
135
+ h.writeUInt32LE(seq, 32);
136
+ h[36] = type;
137
+ h[37] = tableId || 0;
138
+ return h;
139
+ }
140
+
141
+ function finalize(buf) {
142
+ const h = crypto.createHash('sha256').update(buf.subarray(32)).digest();
143
+ h.copy(buf, 0);
144
+ }
145
+
146
+ function verifyHash(buf) {
147
+ const s = buf.slice(0, 32);
148
+ const c = crypto.createHash('sha256').update(buf.subarray(32)).digest();
149
+ return s.equals(c);
150
+ }
151
+
152
+ class JSQLFormat {
153
+ constructor(path) {
154
+ this.path = path;
155
+ this.fd = null;
156
+ this.tables = new Map();
157
+ this.pkIdx = new Map();
158
+ this.nBlocks = 0;
159
+ this.dirty = false;
160
+ this.open = false;
161
+ this._nextTableId = 1;
162
+ }
163
+
164
+ _open() {
165
+ if (this.fd) return;
166
+ this.open = true;
167
+ if (fs.existsSync(this.path)) {
168
+ this.fd = fs.openSync(this.path, 'r+');
169
+ this._load();
170
+ } else {
171
+ this.fd = fs.openSync(this.path, 'w+');
172
+ this._init();
173
+ }
174
+ }
175
+
176
+ _close() {
177
+ if (!this.fd) return;
178
+ if (this.dirty) this._flush();
179
+ fs.closeSync(this.fd);
180
+ this.fd = null;
181
+ this.open = false;
182
+ }
183
+
184
+ _init() {
185
+ const sb = Buffer.alloc(SZ.SUPER);
186
+ sb.write('JSQL', 0, 'utf8');
187
+ // block count = superblock only
188
+ this.nBlocks = 1;
189
+ fs.writeSync(this.fd, sb, 0, SZ.SUPER, 0);
190
+ fs.ftruncateSync(this.fd, SZ.SUPER);
191
+ }
192
+
193
+ _load() {
194
+ const sb = Buffer.alloc(SZ.SUPER);
195
+ fs.readSync(this.fd, sb, 0, SZ.SUPER, 0);
196
+ const magic = sb.toString('utf8', 0, 4);
197
+ if (magic !== 'JSQL') throw new Error('not a .jsql file');
198
+
199
+ const tCount = sb.readUInt16LE(4);
200
+ let off = 6;
201
+ for (let i = 0; i < tCount; i++) {
202
+ const nLen = sb[off]; off += 1;
203
+ const name = sb.toString('utf8', off, off + nLen); off += nLen;
204
+ const sLen = sb.readUInt32LE(off); off += 4;
205
+ const rCnt = sb.readUInt32LE(off); off += 4;
206
+ const pkB = sb.readUInt32LE(off); off += 4;
207
+ const dB = sb.readUInt32LE(off); off += 4;
208
+ const tableId = sb.readUInt16LE(off); off += 2;
209
+ const schema = JSON.parse(sb.toString('utf8', off, off + sLen)); off += sLen;
210
+ this.tables.set(name, { schema, rowCount: rCnt, pkBlock: pkB, dataBlock: dB, tableId });
211
+ this.pkIdx.set(name, this._loadIdx(pkB));
212
+ if ((this.tables.get(name).tableId || 0) >= this._nextTableId)
213
+ this._nextTableId = (this.tables.get(name).tableId || 0) + 1;
214
+ }
215
+
216
+ this.nBlocks = Math.max(1, Math.ceil(fs.fstatSync(this.fd).size / SZ.BLOCK));
217
+ }
218
+
219
+ _saveIdx(name) {
220
+ const entries = this.pkIdx.get(name) || [];
221
+ const maxPerBlock = Math.floor((SZ.DATA - 4) / 12);
222
+ let firstBlock = 0, prevBlock = 0;
223
+
224
+ for (let chunk = 0; chunk < entries.length; chunk += maxPerBlock) {
225
+ const chunkEnd = Math.min(chunk + maxPerBlock, entries.length);
226
+ const buf = Buffer.alloc(SZ.BLOCK);
227
+ mkHeader(BT.IDX, chunk).copy(buf);
228
+ let off = SZ.HEADER;
229
+
230
+ const cnt = chunkEnd - chunk;
231
+ buf.writeUInt32LE(cnt, off); off += 4;
232
+ for (let i = chunk; i < chunkEnd; i++) {
233
+ const e = entries[i];
234
+ buf.writeUInt32LE(e.hash, off); off += 4;
235
+ buf.writeUInt32LE(e.block, off); off += 4;
236
+ buf.writeUInt32LE(e.off, off); off += 4;
237
+ }
238
+
239
+ const bid = this._alloc();
240
+ if (chunk === 0) firstBlock = bid;
241
+ if (prevBlock) buf.writeUInt32LE(bid, 52); // next block pointer
242
+ buf.writeUInt32LE(prevBlock, 56); // prev block pointer
243
+
244
+ // total entry count in first block
245
+ if (chunk === 0) buf.writeUInt32LE(entries.length, 48);
246
+
247
+ finalize(buf);
248
+ fs.writeSync(this.fd, buf, 0, SZ.BLOCK, bid * SZ.BLOCK);
249
+ prevBlock = bid;
250
+ }
251
+
252
+ const t = this.tables.get(name);
253
+ if (t) t.pkBlock = firstBlock;
254
+ return firstBlock;
255
+ }
256
+
257
+ _loadIdx(blockId) {
258
+ if (blockId < 1) return [];
259
+ const all = [];
260
+ let bid = blockId;
261
+ while (bid) {
262
+ const buf = Buffer.alloc(SZ.BLOCK);
263
+ fs.readSync(this.fd, buf, 0, SZ.BLOCK, bid * SZ.BLOCK);
264
+ if (!verifyHash(buf)) throw new Error('corrupt idx block ' + bid);
265
+ let off = SZ.HEADER;
266
+ const cnt = buf.readUInt32LE(off); off += 4;
267
+ for (let i = 0; i < cnt; i++) {
268
+ const hash = buf.readUInt32LE(off); off += 4;
269
+ const block = buf.readUInt32LE(off); off += 4;
270
+ const dataOff = buf.readUInt32LE(off); off += 4;
271
+ all.push({ hash, block, off: dataOff });
272
+ }
273
+ bid = buf.readUInt32LE(52);
274
+ }
275
+ return all;
276
+ }
277
+
278
+ _alloc() { return this.nBlocks++; }
279
+
280
+ _flush() {
281
+ for (const name of this.tables.keys()) this._saveIdx(name);
282
+ this._saveSuper();
283
+ this.dirty = false;
284
+ }
285
+
286
+ _saveSuper() {
287
+ const sb = Buffer.alloc(SZ.SUPER);
288
+ sb.write('JSQL', 0, 'utf8');
289
+ sb.writeUInt16LE(this.tables.size, 4);
290
+ let off = 6;
291
+ for (const [name, t] of this.tables) {
292
+ const nB = Buffer.from(name, 'utf8');
293
+ sb[off] = nB.length; off += 1;
294
+ nB.copy(sb, off); off += nB.length;
295
+ const sStr = JSON.stringify(t.schema);
296
+ sb.writeUInt32LE(sStr.length, off); off += 4;
297
+ sb.writeUInt32LE(t.rowCount, off); off += 4;
298
+ sb.writeUInt32LE(t.pkBlock||0, off); off += 4;
299
+ sb.writeUInt32LE(t.dataBlock||0, off); off += 4;
300
+ sb.writeUInt16LE(t.tableId || 0, off); off += 2;
301
+ Buffer.from(sStr, 'utf8').copy(sb, off); off += sStr.length;
302
+ }
303
+ fs.writeSync(this.fd, sb, 0, SZ.SUPER, 0);
304
+ fs.fsyncSync(this.fd);
305
+ }
306
+
307
+ _reset() {
308
+ fs.ftruncateSync(this.fd, SZ.SUPER);
309
+ const sb = Buffer.alloc(SZ.SUPER); sb.write('JSQL', 0, 'utf8');
310
+ fs.writeSync(this.fd, sb, 0, SZ.SUPER, 0);
311
+ this.nBlocks = 1;
312
+ this._nextTableId = 1;
313
+ this.tables.clear();
314
+ this.pkIdx.clear();
315
+ }
316
+
317
+ saveAll(allTables) {
318
+ this._open();
319
+ this._reset();
320
+
321
+ for (const [name, info] of Object.entries(allTables)) {
322
+ const { schema, rows, pkField } = info;
323
+ if (!rows || rows.length === 0) continue;
324
+
325
+ const tableId = this._nextTableId++;
326
+ this.tables.set(name, { schema, rowCount:0, pkBlock:0, dataBlock:0, tableId });
327
+
328
+ const pkEntries = [];
329
+ const rawBlocks = [];
330
+ const blockIds = [];
331
+ let curBlock = this._alloc();
332
+ let buf = Buffer.alloc(SZ.BLOCK);
333
+ let bp = SZ.HEADER;
334
+ mkHeader(BT.DATA, curBlock, tableId).copy(buf);
335
+
336
+ // Pre-build field encoders once per table
337
+ const encoders = buildEncoders(schema);
338
+ // Scratch buffer for row encoding (max row size fits in one block)
339
+ const scratch = Buffer.allocUnsafe(SZ.DATA);
340
+
341
+ for (const row of rows) {
342
+ const end = encRowInPlace(row, encoders, scratch, 0);
343
+ if (bp + end > SZ.BLOCK) {
344
+ const raw = Buffer.from(buf.subarray(SZ.HEADER, bp));
345
+ rawBlocks.push(raw);
346
+ blockIds.push(curBlock);
347
+ buf.writeUInt32LE(bp - SZ.HEADER, 40);
348
+
349
+ curBlock = this._alloc();
350
+ buf = Buffer.alloc(SZ.BLOCK);
351
+ bp = SZ.HEADER;
352
+ mkHeader(BT.DATA, curBlock, tableId).copy(buf);
353
+ }
354
+
355
+ const startOff = bp;
356
+ scratch.copy(buf, bp, 0, end);
357
+ bp += end;
358
+
359
+ if (pkField && row[pkField] !== undefined) {
360
+ pkEntries.push({ hash: hashPK(row[pkField]), block: curBlock, off: startOff - SZ.HEADER });
361
+ }
362
+ }
363
+
364
+ if (bp > SZ.HEADER) {
365
+ const raw = Buffer.from(buf.subarray(SZ.HEADER, bp));
366
+ rawBlocks.push(raw);
367
+ blockIds.push(curBlock);
368
+ buf.writeUInt32LE(bp - SZ.HEADER, 40);
369
+ }
370
+
371
+ // Compress and write blocks
372
+ for (let i = 0; i < rawBlocks.length; i++) {
373
+ const bid = blockIds[i];
374
+ const compBuf = zlib.gzipSync(rawBlocks[i], { level: 1 });
375
+ const block = Buffer.alloc(SZ.BLOCK);
376
+ mkHeader(BT.DATA, bid, tableId).copy(block);
377
+ block.writeUInt32LE(rawBlocks[i].length, 40);
378
+ block[48] = 1;
379
+ block.writeUInt32LE(compBuf.length, 44);
380
+ compBuf.copy(block, SZ.HEADER);
381
+ finalize(block);
382
+ fs.writeSync(this.fd, block, 0, SZ.BLOCK, bid * SZ.BLOCK);
383
+ }
384
+
385
+ const t = this.tables.get(name);
386
+ t.rowCount = rows.length;
387
+ t.dataBlock = blockIds.length > 0 ? blockIds[blockIds.length - 1] : 0;
388
+
389
+ pkEntries.sort((a, b) => a.hash - b.hash);
390
+ this.pkIdx.set(name, pkEntries);
391
+ this._saveIdx(name);
392
+ }
393
+
394
+ this._saveSuper();
395
+ this.dirty = false;
396
+ }
397
+
398
+ writeTable(name, schema, rows, pkField) {
399
+ this.saveAll({ [name]: { schema, rows, pkField } });
400
+ }
401
+
402
+ readTableSync(name) {
403
+ this._open();
404
+ const t = this.tables.get(name);
405
+ if (!t) return { schema: null, rows: [] };
406
+
407
+ const rows = [];
408
+ for (let bid = 1; bid < this.nBlocks; bid++) {
409
+ const buf = Buffer.alloc(SZ.BLOCK);
410
+ try { fs.readSync(this.fd, buf, 0, SZ.BLOCK, bid * SZ.BLOCK); } catch(e) { continue; }
411
+ if (buf[36] !== BT.DATA || buf[37] !== t.tableId) continue;
412
+ if (!verifyHash(buf)) continue;
413
+
414
+ const compFlag = buf[48];
415
+ const raw = compFlag
416
+ ? zlib.gunzipSync(buf.slice(SZ.HEADER, SZ.HEADER + buf.readUInt32LE(44)))
417
+ : buf.slice(SZ.HEADER, SZ.HEADER + buf.readUInt32LE(40));
418
+
419
+ let off = 0;
420
+ while (off + 4 <= raw.length) {
421
+ const l = raw.readUInt32LE(off);
422
+ if (l === 0 || off + 4 + l > raw.length) break;
423
+ const r = decRow(raw, off);
424
+ rows.push(r.row);
425
+ off = r.off;
426
+ }
427
+ }
428
+ return { schema: t.schema, rows };
429
+ }
430
+
431
+ readTable(name) {
432
+ return this.readTableSync(name);
433
+ }
434
+
435
+ findRow(name, pkValue) {
436
+ const entries = this.pkIdx.get(name);
437
+ if (!entries || entries.length === 0) return null;
438
+ const h = hashPK(pkValue);
439
+ let lo = 0, hi = entries.length - 1;
440
+ while (lo <= hi) {
441
+ const m = (lo + hi) >>> 1;
442
+ const e = entries[m];
443
+ if (e.hash === h) return this._readInBlock(e.block, e.off);
444
+ if (e.hash < h) lo = m + 1;
445
+ else hi = m - 1;
446
+ }
447
+ return null;
448
+ }
449
+
450
+ _readInBlock(blockId, off) {
451
+ const buf = Buffer.alloc(SZ.BLOCK);
452
+ fs.readSync(this.fd, buf, 0, SZ.BLOCK, blockId * SZ.BLOCK);
453
+ const compFlag = buf[48];
454
+ const compSz = buf.readUInt32LE(44);
455
+ const dataSz = buf.readUInt32LE(40);
456
+ let raw;
457
+ if (compFlag) {
458
+ raw = zlib.gunzipSync(buf.slice(SZ.HEADER, SZ.HEADER + compSz));
459
+ } else {
460
+ raw = buf.slice(SZ.HEADER, SZ.HEADER + dataSz);
461
+ }
462
+ const r = decRow(raw, off);
463
+ return r.row;
464
+ }
465
+
466
+ appendRows(name, schema, rows, pkField) {
467
+ this._open();
468
+ const t = this.tables.get(name);
469
+ if (!t) { this.writeTable(name, schema, rows, pkField); return; }
470
+ t.schema = schema;
471
+
472
+ const { rows: oldRows } = this.readTable(name);
473
+ oldRows.push(...rows);
474
+ this.writeTable(name, schema, oldRows, pkField);
475
+ }
476
+ }
477
+
478
+ module.exports = JSQLFormat;
@@ -0,0 +1,448 @@
1
+ const zlib = require('zlib');
2
+ const crypto = require('crypto');
3
+ const fs = require('fs');
4
+
5
+ const MAGIC = Buffer.from('4A534F4C', 'hex');
6
+ const VERSION = 1;
7
+ const BLOCK_SIZE = 65536;
8
+ const BLOCK_HEADER_SIZE = 64;
9
+ const BLOCK_DATA_SIZE = BLOCK_SIZE - BLOCK_HEADER_SIZE;
10
+ const SUPERBLOCK_SIZE = 4096;
11
+
12
+ const BT = { FREE: 0, DATA: 1, PK_INDEX: 2, SUPER: 3 };
13
+
14
+ const TYPE = {
15
+ NULL: 0, BOOL: 1, INT: 2, UINT: 3, FLOAT: 4,
16
+ STRING: 5, BUFFER: 6, ARRAY: 7, OBJECT: 8
17
+ };
18
+
19
+ function typeTag(v) {
20
+ if (v === null || v === undefined) return TYPE.NULL;
21
+ if (typeof v === 'boolean') return TYPE.BOOL;
22
+ if (typeof v === 'number') return Number.isInteger(v) ? TYPE.INT : TYPE.FLOAT;
23
+ if (typeof v === 'string') return TYPE.STRING;
24
+ if (v instanceof Buffer) return TYPE.BUFFER;
25
+ return Array.isArray(v) ? TYPE.ARRAY : TYPE.OBJECT;
26
+ }
27
+
28
+ function encodeValue(v) {
29
+ const tag = typeTag(v);
30
+ switch (tag) {
31
+ case TYPE.NULL: return Buffer.from([tag]);
32
+ case TYPE.BOOL: return Buffer.from([tag, v ? 1 : 0]);
33
+ case TYPE.INT: { const b = Buffer.alloc(5); b[0] = tag; b.writeInt32LE(v, 1); return b; }
34
+ case TYPE.UINT: { const b = Buffer.alloc(5); b[0] = tag; b.writeUInt32LE(v, 1); return b; }
35
+ case TYPE.FLOAT: { const b = Buffer.alloc(9); b[0] = tag; b.writeDoubleLE(v, 1); return b; }
36
+ case TYPE.STRING: { const s = Buffer.from(v, 'utf8'); const b = Buffer.alloc(5 + s.length); b[0] = tag; b.writeUInt32LE(s.length, 1); s.copy(b, 5); return b; }
37
+ case TYPE.BUFFER: { const b = Buffer.alloc(5 + v.length); b[0] = tag; b.writeUInt32LE(v.length, 1); v.copy(b, 5); return b; }
38
+ case TYPE.ARRAY: {
39
+ const parts = [Buffer.from([tag]), encodeValue(v.length)];
40
+ for (const item of v) parts.push(encodeValue(item));
41
+ return Buffer.concat(parts);
42
+ }
43
+ case TYPE.OBJECT: {
44
+ const keys = Object.keys(v);
45
+ const parts = [Buffer.from([tag]), encodeValue(keys.length)];
46
+ for (const k of keys) { parts.push(encodeValue(k)); parts.push(encodeValue(v[k])); }
47
+ return Buffer.concat(parts);
48
+ }
49
+ }
50
+ }
51
+
52
+ function decodeValue(buf, off) {
53
+ const tag = buf[off++];
54
+ switch (tag) {
55
+ case TYPE.NULL: return { v: null, off };
56
+ case TYPE.BOOL: return { v: buf[off] !== 0, off: off + 1 };
57
+ case TYPE.INT: return { v: buf.readInt32LE(off), off: off + 4 };
58
+ case TYPE.UINT: return { v: buf.readUInt32LE(off), off: off + 4 };
59
+ case TYPE.FLOAT: return { v: buf.readDoubleLE(off), off: off + 8 };
60
+ case TYPE.STRING: { const len = buf.readUInt32LE(off); off += 4; return { v: buf.toString('utf8', off, off + len), off: off + len }; }
61
+ case TYPE.BUFFER: { const len = buf.readUInt32LE(off); off += 4; return { v: Buffer.from(buf.subarray(off, off + len)), off: off + len }; }
62
+ case TYPE.ARRAY: {
63
+ let r = decodeValue(buf, off); off = r.off;
64
+ const len = r.v, arr = new Array(len);
65
+ for (let i = 0; i < len; i++) { r = decodeValue(buf, off); arr[i] = r.v; off = r.off; }
66
+ return { v: arr, off };
67
+ }
68
+ case TYPE.OBJECT: {
69
+ let r = decodeValue(buf, off); off = r.off;
70
+ const len = r.v, obj = {};
71
+ for (let i = 0; i < len; i++) { r = decodeValue(buf, off); const k = r.v; off = r.off; r = decodeValue(buf, off); obj[k] = r.v; off = r.off; }
72
+ return { v: obj, off };
73
+ }
74
+ default: throw new Error('Unknown type: ' + tag);
75
+ }
76
+ }
77
+
78
+ function makeBlockHeader(type, seq) {
79
+ const h = Buffer.alloc(BLOCK_HEADER_SIZE);
80
+ h.writeUInt32LE(seq, 32);
81
+ h[36] = type;
82
+ return h;
83
+ }
84
+
85
+ function finalizeBlock(block) {
86
+ const hash = crypto.createHash('sha256').update(block.subarray(32)).digest();
87
+ hash.copy(block, 0);
88
+ return block;
89
+ }
90
+
91
+ function verifyBlockHash(block) {
92
+ const stored = block.slice(0, 32);
93
+ const computed = crypto.createHash('sha256').update(block.subarray(32)).digest();
94
+ return stored.equals(computed);
95
+ }
96
+
97
+ function encodeRow(row, schema) {
98
+ const fields = Object.keys(schema).filter(f => f !== '_softDelete');
99
+ const parts = [encodeValue(fields.length)];
100
+ for (const f of fields) {
101
+ parts.push(encodeValue(f));
102
+ parts.push(encodeValue(row[f]));
103
+ }
104
+ return Buffer.concat(parts);
105
+ }
106
+
107
+ function decodeRow(buf, off) {
108
+ let r = decodeValue(buf, off); off = r.off;
109
+ const fieldCount = r.v;
110
+ const row = {};
111
+ for (let i = 0; i < fieldCount; i++) {
112
+ r = decodeValue(buf, off); const field = r.v; off = r.off;
113
+ r = decodeValue(buf, off); row[field] = r.v; off = r.off;
114
+ }
115
+ return { row, off };
116
+ }
117
+
118
+ class JSQLFile {
119
+ constructor(path, blockSize = BLOCK_SIZE) {
120
+ this.path = path;
121
+ this.blockSize = blockSize;
122
+ this.dataSize = blockSize - BLOCK_HEADER_SIZE;
123
+ this.fd = null;
124
+ this.tables = new Map();
125
+ this.pkIndex = new Map();
126
+ this.totalBlocks = 1;
127
+ this.freeBlocks = [];
128
+ this.dirty = false;
129
+ }
130
+
131
+ _rowSize(row, schema) {
132
+ return encodeRow(row, schema).length + 4;
133
+ }
134
+
135
+ open() {
136
+ if (this.fd) return;
137
+ if (fs.existsSync(this.path)) {
138
+ this.fd = fs.openSync(this.path, 'r+');
139
+ this._load();
140
+ } else {
141
+ this.fd = fs.openSync(this.path, 'w+');
142
+ this._initFile();
143
+ }
144
+ }
145
+
146
+ close() {
147
+ if (!this.fd) return;
148
+ if (this.dirty) this._flush();
149
+ fs.closeSync(this.fd);
150
+ this.fd = null;
151
+ }
152
+
153
+ _initFile() {
154
+ fs.ftruncateSync(this.fd, SUPERBLOCK_SIZE);
155
+ }
156
+
157
+ _load() {
158
+ const sb = Buffer.alloc(SUPERBLOCK_SIZE);
159
+ fs.readSync(this.fd, sb, 0, SUPERBLOCK_SIZE, 0);
160
+ const magic = sb.slice(0, 4);
161
+ if (magic.compare(MAGIC) !== 0) throw new Error('Not a JSQL file');
162
+ const version = sb.readUInt16LE(4);
163
+ this.blockSize = sb.readUInt32LE(6) || BLOCK_SIZE;
164
+ this.dataSize = this.blockSize - BLOCK_HEADER_SIZE;
165
+
166
+ const tableCount = sb.readUInt16LE(10);
167
+ let off = 12;
168
+ for (let i = 0; i < tableCount; i++) {
169
+ const nameLen = Math.min(sb[off], 64);
170
+ const name = sb.toString('utf8', off + 1, off + 1 + nameLen);
171
+ off += 1 + nameLen;
172
+ const schemaLen = sb.readUInt32LE(off); off += 4;
173
+ const rowCount = sb.readUInt32LE(off); off += 4;
174
+ const pkBlock = sb.readUInt32LE(off); off += 4;
175
+ const dataBlock = sb.readUInt32LE(off); off += 4;
176
+ const schemaStr = sb.toString('utf8', off, off + schemaLen);
177
+ off += schemaLen;
178
+
179
+ this.tables.set(name, {
180
+ schema: JSON.parse(schemaStr),
181
+ rowCount, pkBlock, dataBlock
182
+ });
183
+
184
+ // Load PK index
185
+ this.pkIndex.set(name, this._loadPKIndex(pkBlock));
186
+ }
187
+
188
+ this.totalBlocks = Math.max(1, Math.ceil(fs.fstatSync(this.fd).size / this.blockSize));
189
+ }
190
+
191
+ _loadPKIndex(blockId) {
192
+ const entries = [];
193
+ if (blockId < 1 || blockId >= this.totalBlocks) return entries;
194
+ const buf = Buffer.alloc(this.blockSize);
195
+ fs.readSync(this.fd, buf, 0, this.blockSize, blockId * this.blockSize);
196
+ if (!verifyBlockHash(buf)) throw new Error('PK index block corrupted');
197
+
198
+ const type = buf[36];
199
+ if (type !== BT.PK_INDEX) throw new Error('Not a PK index block');
200
+
201
+ let off = BLOCK_HEADER_SIZE;
202
+ const count = buf.readUInt32LE(off); off += 4;
203
+ for (let i = 0; i < count; i++) {
204
+ const pkHash = buf.readBigUInt64LE(off); off += 8;
205
+ const pkOff = buf.readUInt32LE(off); off += 4;
206
+ const rowLen = buf.readUInt32LE(off); off += 4;
207
+ const dataBlock = buf.readUInt32LE(off); off += 4;
208
+ const dataOff = buf.readUInt32LE(off); off += 4;
209
+ entries.push({ pkHash, pkOff, rowLen, dataBlock, dataOff });
210
+ }
211
+ return entries;
212
+ }
213
+
214
+ _savePKIndex(name) {
215
+ const entries = this.pkIndex.get(name) || [];
216
+ const table = this.tables.get(name);
217
+ const buf = Buffer.alloc(this.blockSize);
218
+ makeBlockHeader(BT.PK_INDEX, table ? table.pkBlock : 0).copy(buf);
219
+
220
+ let off = BLOCK_HEADER_SIZE;
221
+ buf.writeUInt32LE(entries.length, off); off += 4;
222
+ for (const e of entries) {
223
+ buf.writeBigUInt64LE(e.pkHash, off); off += 8;
224
+ buf.writeUInt32LE(e.pkOff, off); off += 4;
225
+ buf.writeUInt32LE(e.rowLen, off); off += 4;
226
+ buf.writeUInt32LE(e.dataBlock, off); off += 4;
227
+ buf.writeUInt32LE(e.dataOff, off); off += 4;
228
+ }
229
+ finalizeBlock(buf);
230
+
231
+ const blockId = table ? table.pkBlock : this._allocBlock();
232
+ fs.writeSync(this.fd, buf, 0, this.blockSize, blockId * this.blockSize);
233
+ if (table) table.pkBlock = blockId;
234
+ return blockId;
235
+ }
236
+
237
+ _allocBlock() {
238
+ if (this.freeBlocks.length) return this.freeBlocks.pop();
239
+ return this.totalBlocks++;
240
+ }
241
+
242
+ _flush() {
243
+ // Save all PK indexes and superblock
244
+ for (const name of this.tables.keys()) {
245
+ this._savePKIndex(name);
246
+ }
247
+ this._saveSuperBlock();
248
+ this.dirty = false;
249
+ }
250
+
251
+ _saveSuperBlock() {
252
+ const sb = Buffer.alloc(SUPERBLOCK_SIZE);
253
+ MAGIC.copy(sb, 0);
254
+ sb.writeUInt16LE(VERSION, 4);
255
+ sb.writeUInt32LE(this.blockSize, 6);
256
+ sb.writeUInt16LE(this.tables.size, 10);
257
+
258
+ let off = 12;
259
+ for (const [name, table] of this.tables) {
260
+ const nameBuf = Buffer.from(name, 'utf8');
261
+ sb[off] = nameBuf.length; off += 1;
262
+ nameBuf.copy(sb, off); off += nameBuf.length;
263
+ const schemaStr = JSON.stringify(table.schema);
264
+ sb.writeUInt32LE(schemaStr.length, off); off += 4;
265
+ sb.writeUInt32LE(table.rowCount, off); off += 4;
266
+ sb.writeUInt32LE(table.pkBlock || 0, off); off += 4;
267
+ sb.writeUInt32LE(table.dataBlock || 0, off); off += 4;
268
+ Buffer.from(schemaStr, 'utf8').copy(sb, off); off += schemaStr.length;
269
+ }
270
+
271
+ fs.writeSync(this.fd, sb, 0, SUPERBLOCK_SIZE, 0);
272
+ fs.fsyncSync(this.fd);
273
+ }
274
+
275
+ writeTable(name, schema, rows, pkField) {
276
+ this.open();
277
+ const table = this.tables.get(name) || { schema, rowCount: 0, pkBlock: 0, dataBlock: 0 };
278
+ table.schema = schema;
279
+ this.tables.set(name, table);
280
+
281
+ const pkEntries = [];
282
+ let currentBlock = this._allocBlock();
283
+ let blockBuf = Buffer.alloc(this.blockSize);
284
+ let blockOff = BLOCK_HEADER_SIZE;
285
+ let seq = currentBlock;
286
+ let hasData = false;
287
+
288
+ makeBlockHeader(BT.DATA, seq).copy(blockBuf);
289
+
290
+ for (const row of rows) {
291
+ const rowData = encodeRow(row, schema);
292
+ const rowLen = rowData.length + 4;
293
+ const lenBuf = Buffer.alloc(4);
294
+ lenBuf.writeUInt32LE(rowData.length);
295
+
296
+ if (blockOff + rowLen > this.blockSize) {
297
+ // Flush current block
298
+ blockBuf.writeUInt16LE(0, 38); // rowCount
299
+ // Actually write row count
300
+ // We need to track rows per block properly
301
+ const written = blockOff - BLOCK_HEADER_SIZE;
302
+ blockBuf.writeUInt32LE(written, 40); // data size
303
+
304
+ // Compress
305
+ const rawData = blockBuf.slice(BLOCK_HEADER_SIZE, blockOff);
306
+ const compressed = zlib.gzipSync(rawData, { level: 6 });
307
+ compressed.copy(blockBuf, BLOCK_HEADER_SIZE);
308
+ blockBuf.writeUInt32LE(compressed.length, 44);
309
+ blockBuf[48] = 1; // compressed flag
310
+
311
+ // TODO: track row count properly
312
+ finalizeBlock(blockBuf);
313
+ fs.writeSync(this.fd, blockBuf, 0, this.blockSize, currentBlock * this.blockSize);
314
+
315
+ // New block
316
+ currentBlock = this._allocBlock();
317
+ seq = currentBlock;
318
+ blockBuf = Buffer.alloc(this.blockSize);
319
+ blockOff = BLOCK_HEADER_SIZE;
320
+ makeBlockHeader(BT.DATA, seq).copy(blockBuf);
321
+ }
322
+
323
+ lenBuf.copy(blockBuf, blockOff); blockOff += 4;
324
+ rowData.copy(blockBuf, blockOff); blockOff += rowData.length;
325
+ hasData = true;
326
+
327
+ if (pkField && row[pkField] !== undefined) {
328
+ const pkVal = row[pkField];
329
+ const pkHash = BigInt(hashPK(pkVal));
330
+ pkEntries.push({ pkHash, pkOff: 0, rowLen: rowData.length, dataBlock: currentBlock, dataOff: blockOff - rowData.length - 4, pkVal });
331
+ }
332
+ }
333
+
334
+ if (hasData) {
335
+ const written = blockOff - BLOCK_HEADER_SIZE;
336
+ blockBuf.writeUInt32LE(written, 40);
337
+ const rawData = blockBuf.slice(BLOCK_HEADER_SIZE, blockOff);
338
+ const compressed = zlib.gzipSync(rawData, { level: 6 });
339
+ compressed.copy(blockBuf, BLOCK_HEADER_SIZE);
340
+ blockBuf.writeUInt32LE(compressed.length, 44);
341
+ blockBuf[48] = 1;
342
+ finalizeBlock(blockBuf);
343
+ fs.writeSync(this.fd, blockBuf, 0, this.blockSize, currentBlock * this.blockSize);
344
+ }
345
+
346
+ table.rowCount = rows.length;
347
+ table.dataBlock = seq;
348
+
349
+ // Sort PK entries by hash for binary search
350
+ pkEntries.sort((a, b) => a.pkHash < b.pkHash ? -1 : a.pkHash > b.pkHash ? 1 : 0);
351
+ this.pkIndex.set(name, pkEntries.map(e => ({ pkHash: e.pkHash, pkOff: 0, rowLen: e.rowLen, dataBlock: e.dataBlock, dataOff: e.dataOff })));
352
+ this._savePKIndex(name);
353
+ this._saveSuperBlock();
354
+ this.dirty = false;
355
+ }
356
+
357
+ readTable(name) {
358
+ this.open();
359
+ const table = this.tables.get(name);
360
+ if (!table) return { schema: null, rows: [] };
361
+
362
+ const rows = [];
363
+ const entries = this.pkIndex.get(name) || [];
364
+
365
+ // Collect unique blocks
366
+ const blockMap = new Map();
367
+ for (const e of entries) {
368
+ if (!blockMap.has(e.dataBlock)) blockMap.set(e.dataBlock, new Map());
369
+ blockMap.get(e.dataBlock).set(e.dataOff, e);
370
+ }
371
+
372
+ for (const [blockId, blockEntries] of blockMap) {
373
+ const buf = Buffer.alloc(this.blockSize);
374
+ fs.readSync(this.fd, buf, 0, this.blockSize, blockId * this.blockSize);
375
+ if (!verifyBlockHash(buf)) throw new Error('Block corrupted: ' + blockId);
376
+
377
+ const compressed = buf[48] === 1;
378
+ const compSize = buf.readUInt32LE(44);
379
+ let rawData;
380
+ if (compressed) {
381
+ rawData = zlib.gunzipSync(buf.slice(BLOCK_HEADER_SIZE, BLOCK_HEADER_SIZE + compSize));
382
+ } else {
383
+ rawData = buf.slice(BLOCK_HEADER_SIZE, BLOCK_HEADER_SIZE + buf.readUInt32LE(40));
384
+ }
385
+
386
+ // Read all rows from this block
387
+ let off = 0;
388
+ while (off < rawData.length) {
389
+ const rowLen = rawData.readUInt32LE(off); off += 4;
390
+ if (rowLen === 0 || off + rowLen > rawData.length) break;
391
+ const r = decodeRow(rawData, off);
392
+ rows.push(r.row);
393
+ off += rowLen;
394
+ }
395
+ }
396
+
397
+ return { schema: table.schema, rows };
398
+ }
399
+
400
+ readRowByName(name, pkField, pkValue) {
401
+ this.open();
402
+ const table = this.tables.get(name);
403
+ if (!table) return null;
404
+
405
+ const entries = this.pkIndex.get(name) || [];
406
+ const targetHash = BigInt(hashPK(pkValue));
407
+
408
+ // Binary search
409
+ let lo = 0, hi = entries.length - 1;
410
+ while (lo <= hi) {
411
+ const mid = (lo + hi) >>> 1;
412
+ const e = entries[mid];
413
+ if (e.pkHash === targetHash) return this._readRowAt(e.dataBlock, e.dataOff, e.rowLen);
414
+ if (e.pkHash < targetHash) lo = mid + 1;
415
+ else hi = mid - 1;
416
+ }
417
+ return null;
418
+ }
419
+
420
+ _readRowAt(blockId, dataOff, rowLen) {
421
+ const buf = Buffer.alloc(this.blockSize);
422
+ fs.readSync(this.fd, buf, 0, this.blockSize, blockId * this.blockSize);
423
+
424
+ const compressed = buf[48] === 1;
425
+ const compSize = buf.readUInt32LE(44);
426
+ let rawData;
427
+ if (compressed) {
428
+ rawData = zlib.gunzipSync(buf.slice(BLOCK_HEADER_SIZE, BLOCK_HEADER_SIZE + compSize));
429
+ } else {
430
+ rawData = buf.slice(BLOCK_HEADER_SIZE, BLOCK_HEADER_SIZE + buf.readUInt32LE(40));
431
+ }
432
+
433
+ const off = dataOff - (blockId * this.blockSize + BLOCK_HEADER_SIZE) + BLOCK_HEADER_SIZE;
434
+ const r = decodeRow(rawData, dataOff);
435
+ return r.row;
436
+ }
437
+ }
438
+
439
+ function hashPK(val) {
440
+ const s = String(val);
441
+ let h = 0;
442
+ for (let i = 0; i < s.length; i++) {
443
+ h = (Math.imul(h, 31) + s.charCodeAt(i)) | 0;
444
+ }
445
+ return h >>> 0;
446
+ }
447
+
448
+ module.exports = { JSQLFile, hashPK, encodeRow, decodeRow };
package/lib/table.js CHANGED
@@ -29,6 +29,7 @@ class Table {
29
29
  this._btreesDirty = false;
30
30
 
31
31
  this._primaryKey = null;
32
+ this._pkIndex = null;
32
33
  this._autoIncrementField = null;
33
34
  this._foreignKeys = [];
34
35
  this._checkConstraints = [];
@@ -69,6 +70,7 @@ class Table {
69
70
  this._softDelete = true;
70
71
  }
71
72
 
73
+ this._pkIndex = this._primaryKey ? new Map() : null;
72
74
  this._cachedSchemaFields = Object.keys(schema).filter(f => f !== '_softDelete');
73
75
  this._cachedDateFields = Object.keys(this._dateFields);
74
76
  this._cachedUniqueFields = this._cachedSchemaFields.filter(f => schema[f].unique);
@@ -179,7 +181,12 @@ class Table {
179
181
  } else if (autoIncField && data[autoIncField] > this._autoIncrement) {
180
182
  this._autoIncrement = data[autoIncField];
181
183
  }
184
+ const rowIdx = this._rows.length;
182
185
  this._rows.push(Object.assign({}, data));
186
+ if (this._pkIndex && this._primaryKey) {
187
+ const pkVal = data[this._primaryKey];
188
+ if (pkVal !== undefined && pkVal !== null) this._pkIndex.set(pkVal, rowIdx);
189
+ }
183
190
  items[i] = data;
184
191
  }
185
192
  this._btreesDirty = true;
@@ -222,6 +229,15 @@ class Table {
222
229
 
223
230
  findById(id) {
224
231
  if (!this._primaryKey) throw createError('ER_PARSE_ERROR', 'Table has no primary key');
232
+ // Hash 索引 O(1)
233
+ if (this._pkIndex) {
234
+ const rowIndex = this._pkIndex.get(id);
235
+ if (rowIndex !== undefined) {
236
+ const row = this._rows[rowIndex];
237
+ if (row && !row._deleted) return { ...row };
238
+ }
239
+ return null;
240
+ }
225
241
  this._ensureBTrees();
226
242
  if (this._btrees[this._primaryKey]) {
227
243
  const indices = this._btrees[this._primaryKey].search(id);
@@ -345,6 +361,17 @@ class Table {
345
361
  return this.remove(query);
346
362
  }
347
363
 
364
+ _findRowIndex(id) {
365
+ if (this._pkIndex && this._primaryKey) {
366
+ const idx = this._pkIndex.get(id);
367
+ if (idx !== undefined) return idx;
368
+ }
369
+ for (let i = 0; i < this._rows.length; i++) {
370
+ if (this._rows[i][this._primaryKey] === id) return i;
371
+ }
372
+ return -1;
373
+ }
374
+
348
375
  restore(query = {}) {
349
376
  if (!this._softDelete) throw createError('ER_NOT_SUPPORTED', 'Soft delete is not enabled');
350
377
  const matched = this._applyFilter(this._rows, { ...query, _deleted: true });
@@ -365,6 +392,7 @@ class Table {
365
392
  _truncate() {
366
393
  const count = this._rows.length;
367
394
  this._rows = [];
395
+ if (this._pkIndex) this._pkIndex.clear();
368
396
  for (const key of Object.keys(this._btrees)) {
369
397
  this._btrees[key].clear();
370
398
  }
@@ -902,6 +930,10 @@ class Table {
902
930
 
903
931
  _addToIndexes(rowIndex, data) {
904
932
  this._ensureBTrees();
933
+ if (this._pkIndex && this._primaryKey) {
934
+ const pkVal = data[this._primaryKey];
935
+ if (pkVal !== undefined && pkVal !== null) this._pkIndex.set(pkVal, rowIndex);
936
+ }
905
937
  for (const field of Object.keys(this._indexes)) {
906
938
  const value = data[field];
907
939
  if (value !== undefined) {
@@ -921,6 +953,9 @@ class Table {
921
953
 
922
954
  _removeFromIndexes(rowIndex, data) {
923
955
  this._ensureBTrees();
956
+ if (this._pkIndex && this._primaryKey) {
957
+ this._pkIndex.delete(data[this._primaryKey]);
958
+ }
924
959
  for (const field of Object.keys(this._indexes)) {
925
960
  const value = data[field];
926
961
  if (value !== undefined && this._indexes[field].has(value)) {
@@ -940,8 +975,17 @@ class Table {
940
975
  }
941
976
  }
942
977
 
978
+ _rebuildPKIndex() {
979
+ if (!this._pkIndex || !this._primaryKey) return;
980
+ this._pkIndex.clear();
981
+ for (let i = 0; i < this._rows.length; i++) {
982
+ const pkVal = this._rows[i][this._primaryKey];
983
+ if (pkVal !== undefined && pkVal !== null) this._pkIndex.set(pkVal, i);
984
+ }
985
+ }
986
+
943
987
  _adjustBTreeAfterRemove(removedIndex) {
944
- // 重建受影响的 B-Tree
988
+ this._rebuildPKIndex();
945
989
  for (const field of Object.keys(this._btrees)) {
946
990
  this._rebuildBTree(field);
947
991
  }
@@ -957,6 +1001,7 @@ class Table {
957
1001
  }
958
1002
  }
959
1003
  }
1004
+ this._rebuildPKIndex();
960
1005
  this._rebuildAllBTrees();
961
1006
  for (const field of Object.keys(this._indexes)) {
962
1007
  this.createIndex(field);
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "jsql-neo",
3
- "version": "2.1.4",
3
+ "version": "2.2.1",
4
4
  "description": "JSQL-NEO — Pure JavaScript embedded database with B-Tree indexes, WAL, hash JOIN, Redis-compatible Cache, and MySQL-style error codes",
5
5
  "main": "index.js",
6
6
  "files": [