@liorandb/core 1.0.18 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +39 -13
- package/dist/index.js +458 -117
- package/package.json +1 -1
- package/src/core/checkpoint.ts +163 -0
- package/src/core/compaction.ts +110 -48
- package/src/core/database.ts +101 -61
- package/src/core/wal.ts +268 -0
package/dist/index.js
CHANGED
|
@@ -1,20 +1,18 @@
|
|
|
1
1
|
// src/LioranManager.ts
|
|
2
|
-
import
|
|
3
|
-
import
|
|
2
|
+
import path9 from "path";
|
|
3
|
+
import fs9 from "fs";
|
|
4
4
|
import process2 from "process";
|
|
5
5
|
|
|
6
6
|
// src/core/database.ts
|
|
7
|
-
import
|
|
8
|
-
import
|
|
9
|
-
import { execFile } from "child_process";
|
|
10
|
-
import { promisify } from "util";
|
|
7
|
+
import path6 from "path";
|
|
8
|
+
import fs6 from "fs";
|
|
11
9
|
|
|
12
10
|
// src/core/collection.ts
|
|
13
11
|
import { ClassicLevel as ClassicLevel3 } from "classic-level";
|
|
14
12
|
|
|
15
13
|
// src/core/query.ts
|
|
16
|
-
function getByPath(obj,
|
|
17
|
-
return
|
|
14
|
+
function getByPath(obj, path10) {
|
|
15
|
+
return path10.split(".").reduce((o, p) => o ? o[p] : void 0, obj);
|
|
18
16
|
}
|
|
19
17
|
function matchDocument(doc, query) {
|
|
20
18
|
for (const key of Object.keys(query)) {
|
|
@@ -295,68 +293,97 @@ var Index = class {
|
|
|
295
293
|
|
|
296
294
|
// src/core/compaction.ts
|
|
297
295
|
var TMP_SUFFIX = "__compact_tmp";
|
|
298
|
-
var OLD_SUFFIX = "
|
|
296
|
+
var OLD_SUFFIX = "__compact_old";
|
|
297
|
+
var INDEX_DIR = "__indexes";
|
|
299
298
|
async function compactCollectionEngine(col) {
|
|
300
|
-
await crashRecovery(col.dir);
|
|
301
299
|
const baseDir = col.dir;
|
|
302
300
|
const tmpDir = baseDir + TMP_SUFFIX;
|
|
303
301
|
const oldDir = baseDir + OLD_SUFFIX;
|
|
302
|
+
await crashRecovery(baseDir);
|
|
304
303
|
safeRemove(tmpDir);
|
|
305
304
|
safeRemove(oldDir);
|
|
306
305
|
await snapshotRebuild(col, tmpDir);
|
|
307
|
-
atomicSwap(baseDir, tmpDir, oldDir);
|
|
306
|
+
await atomicSwap(baseDir, tmpDir, oldDir);
|
|
308
307
|
safeRemove(oldDir);
|
|
308
|
+
await reopenCollectionDB(col);
|
|
309
|
+
await rebuildIndexes(col);
|
|
309
310
|
}
|
|
310
311
|
async function snapshotRebuild(col, tmpDir) {
|
|
311
312
|
fs2.mkdirSync(tmpDir, { recursive: true });
|
|
312
|
-
const tmpDB = new ClassicLevel2(tmpDir, {
|
|
313
|
+
const tmpDB = new ClassicLevel2(tmpDir, {
|
|
314
|
+
valueEncoding: "utf8"
|
|
315
|
+
});
|
|
313
316
|
for await (const [key, val] of col.db.iterator()) {
|
|
314
|
-
|
|
317
|
+
if (val !== void 0) {
|
|
318
|
+
await tmpDB.put(key, val);
|
|
319
|
+
}
|
|
315
320
|
}
|
|
316
321
|
await tmpDB.close();
|
|
317
322
|
await col.db.close();
|
|
318
323
|
}
|
|
319
|
-
function atomicSwap(base, tmp, old) {
|
|
324
|
+
async function atomicSwap(base, tmp, old) {
|
|
320
325
|
fs2.renameSync(base, old);
|
|
321
|
-
|
|
326
|
+
try {
|
|
327
|
+
fs2.renameSync(tmp, base);
|
|
328
|
+
} catch (err) {
|
|
329
|
+
if (fs2.existsSync(old)) {
|
|
330
|
+
fs2.renameSync(old, base);
|
|
331
|
+
}
|
|
332
|
+
throw err;
|
|
333
|
+
}
|
|
322
334
|
}
|
|
323
335
|
async function crashRecovery(baseDir) {
|
|
324
336
|
const tmp = baseDir + TMP_SUFFIX;
|
|
325
337
|
const old = baseDir + OLD_SUFFIX;
|
|
326
|
-
|
|
338
|
+
const baseExists = fs2.existsSync(baseDir);
|
|
339
|
+
const tmpExists = fs2.existsSync(tmp);
|
|
340
|
+
const oldExists = fs2.existsSync(old);
|
|
341
|
+
if (tmpExists && oldExists) {
|
|
327
342
|
safeRemove(baseDir);
|
|
328
343
|
fs2.renameSync(tmp, baseDir);
|
|
329
344
|
safeRemove(old);
|
|
345
|
+
return;
|
|
330
346
|
}
|
|
331
|
-
if (
|
|
347
|
+
if (!baseExists && oldExists) {
|
|
332
348
|
fs2.renameSync(old, baseDir);
|
|
349
|
+
return;
|
|
333
350
|
}
|
|
334
|
-
if (
|
|
351
|
+
if (tmpExists && !oldExists) {
|
|
335
352
|
safeRemove(tmp);
|
|
336
353
|
}
|
|
337
354
|
}
|
|
355
|
+
async function reopenCollectionDB(col) {
|
|
356
|
+
col.db = new ClassicLevel2(col.dir, {
|
|
357
|
+
valueEncoding: "utf8"
|
|
358
|
+
});
|
|
359
|
+
}
|
|
338
360
|
async function rebuildIndexes(col) {
|
|
339
|
-
const indexRoot = path2.join(col.dir,
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
for (const idx of col["indexes"].values()) {
|
|
361
|
+
const indexRoot = path2.join(col.dir, INDEX_DIR);
|
|
362
|
+
const oldIndexes = new Map(col["indexes"]);
|
|
363
|
+
for (const idx of oldIndexes.values()) {
|
|
343
364
|
try {
|
|
344
365
|
await idx.close();
|
|
345
366
|
} catch {
|
|
346
367
|
}
|
|
347
368
|
}
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
369
|
+
safeRemove(indexRoot);
|
|
370
|
+
fs2.mkdirSync(indexRoot, { recursive: true });
|
|
371
|
+
const rebuiltIndexes = /* @__PURE__ */ new Map();
|
|
372
|
+
for (const idx of oldIndexes.values()) {
|
|
373
|
+
const rebuilt = new Index(col.dir, idx.field, {
|
|
351
374
|
unique: idx.unique
|
|
352
375
|
});
|
|
353
376
|
for await (const [, enc] of col.db.iterator()) {
|
|
354
|
-
|
|
355
|
-
|
|
377
|
+
if (!enc) continue;
|
|
378
|
+
try {
|
|
379
|
+
const doc = decryptData(enc);
|
|
380
|
+
await rebuilt.insert(doc);
|
|
381
|
+
} catch {
|
|
382
|
+
}
|
|
356
383
|
}
|
|
357
|
-
|
|
384
|
+
rebuiltIndexes.set(idx.field, rebuilt);
|
|
358
385
|
}
|
|
359
|
-
col["indexes"] =
|
|
386
|
+
col["indexes"] = rebuiltIndexes;
|
|
360
387
|
}
|
|
361
388
|
function safeRemove(p) {
|
|
362
389
|
if (fs2.existsSync(p)) {
|
|
@@ -820,10 +847,305 @@ var MigrationEngine = class {
|
|
|
820
847
|
}
|
|
821
848
|
};
|
|
822
849
|
|
|
850
|
+
// src/core/wal.ts
|
|
851
|
+
import fs4 from "fs";
|
|
852
|
+
import path4 from "path";
|
|
853
|
+
var MAX_WAL_SIZE = 16 * 1024 * 1024;
|
|
854
|
+
var WAL_DIR = "__wal";
|
|
855
|
+
var CRC32_TABLE = (() => {
|
|
856
|
+
const table = new Uint32Array(256);
|
|
857
|
+
for (let i = 0; i < 256; i++) {
|
|
858
|
+
let c = i;
|
|
859
|
+
for (let k = 0; k < 8; k++) {
|
|
860
|
+
c = c & 1 ? 3988292384 ^ c >>> 1 : c >>> 1;
|
|
861
|
+
}
|
|
862
|
+
table[i] = c >>> 0;
|
|
863
|
+
}
|
|
864
|
+
return table;
|
|
865
|
+
})();
|
|
866
|
+
function crc32(input) {
|
|
867
|
+
let crc = 4294967295;
|
|
868
|
+
for (let i = 0; i < input.length; i++) {
|
|
869
|
+
crc = CRC32_TABLE[(crc ^ input.charCodeAt(i)) & 255] ^ crc >>> 8;
|
|
870
|
+
}
|
|
871
|
+
return (crc ^ 4294967295) >>> 0;
|
|
872
|
+
}
|
|
873
|
+
var WALManager = class {
|
|
874
|
+
walDir;
|
|
875
|
+
currentGen = 1;
|
|
876
|
+
lsn = 0;
|
|
877
|
+
fd = null;
|
|
878
|
+
constructor(baseDir) {
|
|
879
|
+
this.walDir = path4.join(baseDir, WAL_DIR);
|
|
880
|
+
fs4.mkdirSync(this.walDir, { recursive: true });
|
|
881
|
+
this.currentGen = this.detectLastGeneration();
|
|
882
|
+
this.recoverLSNFromExistingLogs();
|
|
883
|
+
}
|
|
884
|
+
/* -------------------------
|
|
885
|
+
INTERNAL HELPERS
|
|
886
|
+
------------------------- */
|
|
887
|
+
walPath(gen = this.currentGen) {
|
|
888
|
+
return path4.join(
|
|
889
|
+
this.walDir,
|
|
890
|
+
`wal-${String(gen).padStart(6, "0")}.log`
|
|
891
|
+
);
|
|
892
|
+
}
|
|
893
|
+
detectLastGeneration() {
|
|
894
|
+
if (!fs4.existsSync(this.walDir)) return 1;
|
|
895
|
+
const files = fs4.readdirSync(this.walDir);
|
|
896
|
+
let max = 0;
|
|
897
|
+
for (const f of files) {
|
|
898
|
+
const m = f.match(/^wal-(\d+)\.log$/);
|
|
899
|
+
if (m) {
|
|
900
|
+
const gen = Number(m[1]);
|
|
901
|
+
if (!Number.isNaN(gen)) {
|
|
902
|
+
max = Math.max(max, gen);
|
|
903
|
+
}
|
|
904
|
+
}
|
|
905
|
+
}
|
|
906
|
+
return max || 1;
|
|
907
|
+
}
|
|
908
|
+
recoverLSNFromExistingLogs() {
|
|
909
|
+
const files = this.getSortedWalFiles();
|
|
910
|
+
for (const file of files) {
|
|
911
|
+
const filePath = path4.join(this.walDir, file);
|
|
912
|
+
const lines = fs4.readFileSync(filePath, "utf8").split("\n");
|
|
913
|
+
for (const line of lines) {
|
|
914
|
+
if (!line.trim()) continue;
|
|
915
|
+
try {
|
|
916
|
+
const parsed = JSON.parse(line);
|
|
917
|
+
const { crc, ...record } = parsed;
|
|
918
|
+
if (crc32(JSON.stringify(record)) !== crc) break;
|
|
919
|
+
this.lsn = Math.max(this.lsn, record.lsn);
|
|
920
|
+
} catch {
|
|
921
|
+
break;
|
|
922
|
+
}
|
|
923
|
+
}
|
|
924
|
+
}
|
|
925
|
+
}
|
|
926
|
+
getSortedWalFiles() {
|
|
927
|
+
return fs4.readdirSync(this.walDir).filter((f) => /^wal-\d+\.log$/.test(f)).sort((a, b) => {
|
|
928
|
+
const ga = Number(a.match(/^wal-(\d+)\.log$/)[1]);
|
|
929
|
+
const gb = Number(b.match(/^wal-(\d+)\.log$/)[1]);
|
|
930
|
+
return ga - gb;
|
|
931
|
+
});
|
|
932
|
+
}
|
|
933
|
+
async open() {
|
|
934
|
+
if (!this.fd) {
|
|
935
|
+
this.fd = await fs4.promises.open(this.walPath(), "a");
|
|
936
|
+
}
|
|
937
|
+
}
|
|
938
|
+
async rotate() {
|
|
939
|
+
if (this.fd) {
|
|
940
|
+
await this.fd.sync();
|
|
941
|
+
await this.fd.close();
|
|
942
|
+
this.fd = null;
|
|
943
|
+
}
|
|
944
|
+
this.currentGen++;
|
|
945
|
+
}
|
|
946
|
+
/* -------------------------
|
|
947
|
+
APPEND (Crash-safe)
|
|
948
|
+
------------------------- */
|
|
949
|
+
async append(record) {
|
|
950
|
+
await this.open();
|
|
951
|
+
const full = {
|
|
952
|
+
...record,
|
|
953
|
+
lsn: ++this.lsn
|
|
954
|
+
};
|
|
955
|
+
const body = JSON.stringify(full);
|
|
956
|
+
const stored = {
|
|
957
|
+
...full,
|
|
958
|
+
crc: crc32(body)
|
|
959
|
+
};
|
|
960
|
+
const line = JSON.stringify(stored) + "\n";
|
|
961
|
+
await this.fd.write(line);
|
|
962
|
+
await this.fd.sync();
|
|
963
|
+
const stat = await this.fd.stat();
|
|
964
|
+
if (stat.size >= MAX_WAL_SIZE) {
|
|
965
|
+
await this.rotate();
|
|
966
|
+
}
|
|
967
|
+
return full.lsn;
|
|
968
|
+
}
|
|
969
|
+
/* -------------------------
|
|
970
|
+
REPLAY (Auto-heal tail)
|
|
971
|
+
------------------------- */
|
|
972
|
+
async replay(fromLSN, apply) {
|
|
973
|
+
if (!fs4.existsSync(this.walDir)) return;
|
|
974
|
+
const files = this.getSortedWalFiles();
|
|
975
|
+
for (const file of files) {
|
|
976
|
+
const filePath = path4.join(this.walDir, file);
|
|
977
|
+
const fd = fs4.openSync(filePath, "r+");
|
|
978
|
+
const content = fs4.readFileSync(filePath, "utf8");
|
|
979
|
+
const lines = content.split("\n");
|
|
980
|
+
let validOffset = 0;
|
|
981
|
+
for (let i = 0; i < lines.length; i++) {
|
|
982
|
+
const line = lines[i];
|
|
983
|
+
if (!line.trim()) {
|
|
984
|
+
validOffset += line.length + 1;
|
|
985
|
+
continue;
|
|
986
|
+
}
|
|
987
|
+
let parsed;
|
|
988
|
+
try {
|
|
989
|
+
parsed = JSON.parse(line);
|
|
990
|
+
} catch {
|
|
991
|
+
break;
|
|
992
|
+
}
|
|
993
|
+
const { crc, ...record } = parsed;
|
|
994
|
+
const expected = crc32(JSON.stringify(record));
|
|
995
|
+
if (expected !== crc) {
|
|
996
|
+
break;
|
|
997
|
+
}
|
|
998
|
+
validOffset += line.length + 1;
|
|
999
|
+
if (record.lsn <= fromLSN) continue;
|
|
1000
|
+
this.lsn = Math.max(this.lsn, record.lsn);
|
|
1001
|
+
await apply(record);
|
|
1002
|
+
}
|
|
1003
|
+
const stat = fs4.fstatSync(fd);
|
|
1004
|
+
if (validOffset < stat.size) {
|
|
1005
|
+
fs4.ftruncateSync(fd, validOffset);
|
|
1006
|
+
}
|
|
1007
|
+
fs4.closeSync(fd);
|
|
1008
|
+
}
|
|
1009
|
+
}
|
|
1010
|
+
/* -------------------------
|
|
1011
|
+
CLEANUP
|
|
1012
|
+
------------------------- */
|
|
1013
|
+
async cleanup(beforeGen) {
|
|
1014
|
+
if (!fs4.existsSync(this.walDir)) return;
|
|
1015
|
+
const files = fs4.readdirSync(this.walDir);
|
|
1016
|
+
for (const f of files) {
|
|
1017
|
+
const m = f.match(/^wal-(\d+)\.log$/);
|
|
1018
|
+
if (!m) continue;
|
|
1019
|
+
const gen = Number(m[1]);
|
|
1020
|
+
if (gen < beforeGen) {
|
|
1021
|
+
fs4.unlinkSync(path4.join(this.walDir, f));
|
|
1022
|
+
}
|
|
1023
|
+
}
|
|
1024
|
+
}
|
|
1025
|
+
/* -------------------------
|
|
1026
|
+
GETTERS
|
|
1027
|
+
------------------------- */
|
|
1028
|
+
getCurrentLSN() {
|
|
1029
|
+
return this.lsn;
|
|
1030
|
+
}
|
|
1031
|
+
getCurrentGen() {
|
|
1032
|
+
return this.currentGen;
|
|
1033
|
+
}
|
|
1034
|
+
};
|
|
1035
|
+
|
|
1036
|
+
// src/core/checkpoint.ts
|
|
1037
|
+
import fs5 from "fs";
|
|
1038
|
+
import path5 from "path";
|
|
1039
|
+
var CHECKPOINT_A = "__checkpoint_A.json";
|
|
1040
|
+
var CHECKPOINT_B = "__checkpoint_B.json";
|
|
1041
|
+
var FORMAT_VERSION = 1;
|
|
1042
|
+
var CRC32_TABLE2 = (() => {
|
|
1043
|
+
const table = new Uint32Array(256);
|
|
1044
|
+
for (let i = 0; i < 256; i++) {
|
|
1045
|
+
let c = i;
|
|
1046
|
+
for (let k = 0; k < 8; k++) {
|
|
1047
|
+
c = c & 1 ? 3988292384 ^ c >>> 1 : c >>> 1;
|
|
1048
|
+
}
|
|
1049
|
+
table[i] = c >>> 0;
|
|
1050
|
+
}
|
|
1051
|
+
return table;
|
|
1052
|
+
})();
|
|
1053
|
+
function crc322(input) {
|
|
1054
|
+
let crc = 4294967295;
|
|
1055
|
+
for (let i = 0; i < input.length; i++) {
|
|
1056
|
+
crc = CRC32_TABLE2[(crc ^ input.charCodeAt(i)) & 255] ^ crc >>> 8;
|
|
1057
|
+
}
|
|
1058
|
+
return (crc ^ 4294967295) >>> 0;
|
|
1059
|
+
}
|
|
1060
|
+
var CheckpointManager = class {
|
|
1061
|
+
baseDir;
|
|
1062
|
+
data;
|
|
1063
|
+
constructor(baseDir) {
|
|
1064
|
+
this.baseDir = baseDir;
|
|
1065
|
+
this.data = {
|
|
1066
|
+
lsn: 0,
|
|
1067
|
+
walGen: 1,
|
|
1068
|
+
time: 0,
|
|
1069
|
+
version: FORMAT_VERSION
|
|
1070
|
+
};
|
|
1071
|
+
this.load();
|
|
1072
|
+
}
|
|
1073
|
+
/* -------------------------
|
|
1074
|
+
LOAD (CRC + FALLBACK)
|
|
1075
|
+
------------------------- */
|
|
1076
|
+
load() {
|
|
1077
|
+
const a = this.readCheckpoint(CHECKPOINT_A);
|
|
1078
|
+
const b = this.readCheckpoint(CHECKPOINT_B);
|
|
1079
|
+
if (a && b) {
|
|
1080
|
+
this.data = a.data.lsn >= b.data.lsn ? a.data : b.data;
|
|
1081
|
+
return;
|
|
1082
|
+
}
|
|
1083
|
+
if (a) {
|
|
1084
|
+
this.data = a.data;
|
|
1085
|
+
return;
|
|
1086
|
+
}
|
|
1087
|
+
if (b) {
|
|
1088
|
+
this.data = b.data;
|
|
1089
|
+
return;
|
|
1090
|
+
}
|
|
1091
|
+
console.warn("No valid checkpoint found, starting from zero");
|
|
1092
|
+
}
|
|
1093
|
+
readCheckpoint(file) {
|
|
1094
|
+
const filePath = path5.join(this.baseDir, file);
|
|
1095
|
+
if (!fs5.existsSync(filePath)) return null;
|
|
1096
|
+
try {
|
|
1097
|
+
const raw = fs5.readFileSync(filePath, "utf8");
|
|
1098
|
+
const parsed = JSON.parse(raw);
|
|
1099
|
+
if (!parsed?.data || typeof parsed.crc !== "number") {
|
|
1100
|
+
return null;
|
|
1101
|
+
}
|
|
1102
|
+
const expected = crc322(JSON.stringify(parsed.data));
|
|
1103
|
+
if (expected !== parsed.crc) {
|
|
1104
|
+
console.error(`Checkpoint CRC mismatch: ${file}`);
|
|
1105
|
+
return null;
|
|
1106
|
+
}
|
|
1107
|
+
return parsed;
|
|
1108
|
+
} catch {
|
|
1109
|
+
return null;
|
|
1110
|
+
}
|
|
1111
|
+
}
|
|
1112
|
+
/* -------------------------
|
|
1113
|
+
SAVE (DUAL WRITE)
|
|
1114
|
+
------------------------- */
|
|
1115
|
+
save(lsn, walGen) {
|
|
1116
|
+
const data = {
|
|
1117
|
+
lsn,
|
|
1118
|
+
walGen,
|
|
1119
|
+
time: Date.now(),
|
|
1120
|
+
version: FORMAT_VERSION
|
|
1121
|
+
};
|
|
1122
|
+
const stored = {
|
|
1123
|
+
data,
|
|
1124
|
+
crc: crc322(JSON.stringify(data))
|
|
1125
|
+
};
|
|
1126
|
+
const target = lsn % 2 === 0 ? CHECKPOINT_A : CHECKPOINT_B;
|
|
1127
|
+
try {
|
|
1128
|
+
fs5.writeFileSync(
|
|
1129
|
+
path5.join(this.baseDir, target),
|
|
1130
|
+
JSON.stringify(stored, null, 2),
|
|
1131
|
+
"utf8"
|
|
1132
|
+
);
|
|
1133
|
+
this.data = data;
|
|
1134
|
+
} catch (err) {
|
|
1135
|
+
console.error("Failed to write checkpoint:", err);
|
|
1136
|
+
}
|
|
1137
|
+
}
|
|
1138
|
+
/* -------------------------
|
|
1139
|
+
GET CURRENT
|
|
1140
|
+
------------------------- */
|
|
1141
|
+
get() {
|
|
1142
|
+
return this.data;
|
|
1143
|
+
}
|
|
1144
|
+
};
|
|
1145
|
+
|
|
823
1146
|
// src/core/database.ts
|
|
824
|
-
var exec = promisify(execFile);
|
|
825
1147
|
var META_FILE = "__db_meta.json";
|
|
826
|
-
var META_VERSION =
|
|
1148
|
+
var META_VERSION = 2;
|
|
827
1149
|
var DEFAULT_SCHEMA_VERSION = "v1";
|
|
828
1150
|
var DBTransactionContext = class {
|
|
829
1151
|
constructor(db, txId) {
|
|
@@ -846,11 +1168,24 @@ var DBTransactionContext = class {
|
|
|
846
1168
|
});
|
|
847
1169
|
}
|
|
848
1170
|
async commit() {
|
|
849
|
-
|
|
850
|
-
|
|
1171
|
+
for (const op of this.ops) {
|
|
1172
|
+
await this.db.wal.append({
|
|
1173
|
+
tx: this.txId,
|
|
1174
|
+
type: "op",
|
|
1175
|
+
payload: op
|
|
1176
|
+
});
|
|
1177
|
+
}
|
|
1178
|
+
const commitLSN = await this.db.wal.append({
|
|
1179
|
+
tx: this.txId,
|
|
1180
|
+
type: "commit"
|
|
1181
|
+
});
|
|
851
1182
|
await this.db.applyTransaction(this.ops);
|
|
852
|
-
await this.db.
|
|
853
|
-
|
|
1183
|
+
const appliedLSN = await this.db.wal.append({
|
|
1184
|
+
tx: this.txId,
|
|
1185
|
+
type: "applied"
|
|
1186
|
+
});
|
|
1187
|
+
this.db.advanceCheckpoint(appliedLSN);
|
|
1188
|
+
await this.db.postCommitMaintenance();
|
|
854
1189
|
}
|
|
855
1190
|
};
|
|
856
1191
|
var LioranDB = class _LioranDB {
|
|
@@ -858,26 +1193,68 @@ var LioranDB = class _LioranDB {
|
|
|
858
1193
|
dbName;
|
|
859
1194
|
manager;
|
|
860
1195
|
collections;
|
|
861
|
-
walPath;
|
|
862
1196
|
metaPath;
|
|
863
1197
|
meta;
|
|
864
1198
|
migrator;
|
|
865
1199
|
static TX_SEQ = 0;
|
|
1200
|
+
wal;
|
|
1201
|
+
checkpoint;
|
|
866
1202
|
constructor(basePath, dbName, manager) {
|
|
867
1203
|
this.basePath = basePath;
|
|
868
1204
|
this.dbName = dbName;
|
|
869
1205
|
this.manager = manager;
|
|
870
1206
|
this.collections = /* @__PURE__ */ new Map();
|
|
871
|
-
this.
|
|
872
|
-
|
|
873
|
-
fs4.mkdirSync(basePath, { recursive: true });
|
|
1207
|
+
this.metaPath = path6.join(basePath, META_FILE);
|
|
1208
|
+
fs6.mkdirSync(basePath, { recursive: true });
|
|
874
1209
|
this.loadMeta();
|
|
1210
|
+
this.wal = new WALManager(basePath);
|
|
1211
|
+
this.checkpoint = new CheckpointManager(basePath);
|
|
875
1212
|
this.migrator = new MigrationEngine(this);
|
|
876
|
-
this.
|
|
1213
|
+
this.initialize().catch(console.error);
|
|
1214
|
+
}
|
|
1215
|
+
/* ------------------------- INIT & RECOVERY ------------------------- */
|
|
1216
|
+
async initialize() {
|
|
1217
|
+
await this.recoverFromWAL();
|
|
1218
|
+
}
|
|
1219
|
+
async recoverFromWAL() {
|
|
1220
|
+
const checkpointData = this.checkpoint.get();
|
|
1221
|
+
const fromLSN = checkpointData.lsn;
|
|
1222
|
+
const committed = /* @__PURE__ */ new Set();
|
|
1223
|
+
const applied = /* @__PURE__ */ new Set();
|
|
1224
|
+
const ops = /* @__PURE__ */ new Map();
|
|
1225
|
+
await this.wal.replay(fromLSN, async (record) => {
|
|
1226
|
+
if (record.type === "commit") {
|
|
1227
|
+
committed.add(record.tx);
|
|
1228
|
+
} else if (record.type === "applied") {
|
|
1229
|
+
applied.add(record.tx);
|
|
1230
|
+
} else if (record.type === "op") {
|
|
1231
|
+
if (!ops.has(record.tx)) ops.set(record.tx, []);
|
|
1232
|
+
ops.get(record.tx).push(record.payload);
|
|
1233
|
+
}
|
|
1234
|
+
});
|
|
1235
|
+
let highestAppliedLSN = fromLSN;
|
|
1236
|
+
for (const tx of committed) {
|
|
1237
|
+
if (applied.has(tx)) continue;
|
|
1238
|
+
const txOps = ops.get(tx);
|
|
1239
|
+
if (txOps) {
|
|
1240
|
+
await this.applyTransaction(txOps);
|
|
1241
|
+
highestAppliedLSN = this.wal.getCurrentLSN();
|
|
1242
|
+
}
|
|
1243
|
+
}
|
|
1244
|
+
this.advanceCheckpoint(highestAppliedLSN);
|
|
1245
|
+
}
|
|
1246
|
+
/* ------------------------- CHECKPOINT ADVANCE ------------------------- */
|
|
1247
|
+
advanceCheckpoint(lsn) {
|
|
1248
|
+
const current = this.checkpoint.get();
|
|
1249
|
+
if (lsn > current.lsn) {
|
|
1250
|
+
this.checkpoint.save(lsn, this.wal.getCurrentGen());
|
|
1251
|
+
this.wal.cleanup(this.wal.getCurrentGen() - 1).catch(() => {
|
|
1252
|
+
});
|
|
1253
|
+
}
|
|
877
1254
|
}
|
|
878
1255
|
/* ------------------------- META ------------------------- */
|
|
879
1256
|
loadMeta() {
|
|
880
|
-
if (!
|
|
1257
|
+
if (!fs6.existsSync(this.metaPath)) {
|
|
881
1258
|
this.meta = {
|
|
882
1259
|
version: META_VERSION,
|
|
883
1260
|
indexes: {},
|
|
@@ -886,14 +1263,14 @@ var LioranDB = class _LioranDB {
|
|
|
886
1263
|
this.saveMeta();
|
|
887
1264
|
return;
|
|
888
1265
|
}
|
|
889
|
-
this.meta = JSON.parse(
|
|
1266
|
+
this.meta = JSON.parse(fs6.readFileSync(this.metaPath, "utf8"));
|
|
890
1267
|
if (!this.meta.schemaVersion) {
|
|
891
1268
|
this.meta.schemaVersion = DEFAULT_SCHEMA_VERSION;
|
|
892
1269
|
this.saveMeta();
|
|
893
1270
|
}
|
|
894
1271
|
}
|
|
895
1272
|
saveMeta() {
|
|
896
|
-
|
|
1273
|
+
fs6.writeFileSync(this.metaPath, JSON.stringify(this.meta, null, 2));
|
|
897
1274
|
}
|
|
898
1275
|
getSchemaVersion() {
|
|
899
1276
|
return this.meta.schemaVersion;
|
|
@@ -912,44 +1289,7 @@ var LioranDB = class _LioranDB {
|
|
|
912
1289
|
async applyMigrations(targetVersion) {
|
|
913
1290
|
await this.migrator.upgradeToLatest();
|
|
914
1291
|
}
|
|
915
|
-
/* -------------------------
|
|
916
|
-
async writeWAL(entries) {
|
|
917
|
-
const fd = await fs4.promises.open(this.walPath, "a");
|
|
918
|
-
for (const e of entries) {
|
|
919
|
-
await fd.write(JSON.stringify(e) + "\n");
|
|
920
|
-
}
|
|
921
|
-
await fd.sync();
|
|
922
|
-
await fd.close();
|
|
923
|
-
}
|
|
924
|
-
async clearWAL() {
|
|
925
|
-
try {
|
|
926
|
-
await fs4.promises.unlink(this.walPath);
|
|
927
|
-
} catch {
|
|
928
|
-
}
|
|
929
|
-
}
|
|
930
|
-
async recoverFromWAL() {
|
|
931
|
-
if (!fs4.existsSync(this.walPath)) return;
|
|
932
|
-
const raw = await fs4.promises.readFile(this.walPath, "utf8");
|
|
933
|
-
const committed = /* @__PURE__ */ new Set();
|
|
934
|
-
const applied = /* @__PURE__ */ new Set();
|
|
935
|
-
const ops = /* @__PURE__ */ new Map();
|
|
936
|
-
for (const line of raw.split("\n")) {
|
|
937
|
-
if (!line.trim()) continue;
|
|
938
|
-
const entry = JSON.parse(line);
|
|
939
|
-
if ("commit" in entry) committed.add(entry.tx);
|
|
940
|
-
else if ("applied" in entry) applied.add(entry.tx);
|
|
941
|
-
else {
|
|
942
|
-
if (!ops.has(entry.tx)) ops.set(entry.tx, []);
|
|
943
|
-
ops.get(entry.tx).push(entry);
|
|
944
|
-
}
|
|
945
|
-
}
|
|
946
|
-
for (const tx of committed) {
|
|
947
|
-
if (applied.has(tx)) continue;
|
|
948
|
-
const txOps = ops.get(tx);
|
|
949
|
-
if (txOps) await this.applyTransaction(txOps);
|
|
950
|
-
}
|
|
951
|
-
await this.clearWAL();
|
|
952
|
-
}
|
|
1292
|
+
/* ------------------------- TX APPLY ------------------------- */
|
|
953
1293
|
async applyTransaction(ops) {
|
|
954
1294
|
for (const { col, op, args } of ops) {
|
|
955
1295
|
const collection = this.collection(col);
|
|
@@ -965,8 +1305,8 @@ var LioranDB = class _LioranDB {
|
|
|
965
1305
|
}
|
|
966
1306
|
return col2;
|
|
967
1307
|
}
|
|
968
|
-
const colPath =
|
|
969
|
-
|
|
1308
|
+
const colPath = path6.join(this.basePath, name);
|
|
1309
|
+
fs6.mkdirSync(colPath, { recursive: true });
|
|
970
1310
|
const col = new Collection(
|
|
971
1311
|
colPath,
|
|
972
1312
|
schema,
|
|
@@ -1004,12 +1344,10 @@ var LioranDB = class _LioranDB {
|
|
|
1004
1344
|
}
|
|
1005
1345
|
/* ------------------------- COMPACTION ------------------------- */
|
|
1006
1346
|
async compactCollection(name) {
|
|
1007
|
-
await this.clearWAL();
|
|
1008
1347
|
const col = this.collection(name);
|
|
1009
1348
|
await col.compact();
|
|
1010
1349
|
}
|
|
1011
1350
|
async compactAll() {
|
|
1012
|
-
await this.clearWAL();
|
|
1013
1351
|
for (const name of this.collections.keys()) {
|
|
1014
1352
|
await this.compactCollection(name);
|
|
1015
1353
|
}
|
|
@@ -1022,6 +1360,9 @@ var LioranDB = class _LioranDB {
|
|
|
1022
1360
|
await tx.commit();
|
|
1023
1361
|
return result;
|
|
1024
1362
|
}
|
|
1363
|
+
/* ------------------------- POST COMMIT ------------------------- */
|
|
1364
|
+
async postCommitMaintenance() {
|
|
1365
|
+
}
|
|
1025
1366
|
/* ------------------------- SHUTDOWN ------------------------- */
|
|
1026
1367
|
async close() {
|
|
1027
1368
|
for (const col of this.collections.values()) {
|
|
@@ -1036,15 +1377,15 @@ var LioranDB = class _LioranDB {
|
|
|
1036
1377
|
|
|
1037
1378
|
// src/utils/rootpath.ts
|
|
1038
1379
|
import os2 from "os";
|
|
1039
|
-
import
|
|
1040
|
-
import
|
|
1380
|
+
import path7 from "path";
|
|
1381
|
+
import fs7 from "fs";
|
|
1041
1382
|
function getDefaultRootPath() {
|
|
1042
1383
|
let dbPath = process.env.LIORANDB_PATH;
|
|
1043
1384
|
if (!dbPath) {
|
|
1044
1385
|
const homeDir = os2.homedir();
|
|
1045
|
-
dbPath =
|
|
1046
|
-
if (!
|
|
1047
|
-
|
|
1386
|
+
dbPath = path7.join(homeDir, "LioranDB", "db");
|
|
1387
|
+
if (!fs7.existsSync(dbPath)) {
|
|
1388
|
+
fs7.mkdirSync(dbPath, { recursive: true });
|
|
1048
1389
|
}
|
|
1049
1390
|
process.env.LIORANDB_PATH = dbPath;
|
|
1050
1391
|
}
|
|
@@ -1059,24 +1400,24 @@ import net from "net";
|
|
|
1059
1400
|
|
|
1060
1401
|
// src/ipc/socketPath.ts
|
|
1061
1402
|
import os3 from "os";
|
|
1062
|
-
import
|
|
1403
|
+
import path8 from "path";
|
|
1063
1404
|
function getIPCSocketPath(rootPath) {
|
|
1064
1405
|
if (os3.platform() === "win32") {
|
|
1065
1406
|
return `\\\\.\\pipe\\liorandb_${rootPath.replace(/[:\\\/]/g, "_")}`;
|
|
1066
1407
|
}
|
|
1067
|
-
return
|
|
1408
|
+
return path8.join(rootPath, ".lioran.sock");
|
|
1068
1409
|
}
|
|
1069
1410
|
|
|
1070
1411
|
// src/ipc/client.ts
|
|
1071
1412
|
function delay(ms) {
|
|
1072
1413
|
return new Promise((r) => setTimeout(r, ms));
|
|
1073
1414
|
}
|
|
1074
|
-
async function connectWithRetry(
|
|
1415
|
+
async function connectWithRetry(path10) {
|
|
1075
1416
|
let attempt = 0;
|
|
1076
1417
|
while (true) {
|
|
1077
1418
|
try {
|
|
1078
1419
|
return await new Promise((resolve, reject) => {
|
|
1079
|
-
const socket = net.connect(
|
|
1420
|
+
const socket = net.connect(path10, () => resolve(socket));
|
|
1080
1421
|
socket.once("error", reject);
|
|
1081
1422
|
});
|
|
1082
1423
|
} catch (err) {
|
|
@@ -1159,11 +1500,11 @@ var DBQueue = class {
|
|
|
1159
1500
|
return this.exec("compact:all", {});
|
|
1160
1501
|
}
|
|
1161
1502
|
/* ----------------------------- SNAPSHOT API ----------------------------- */
|
|
1162
|
-
snapshot(
|
|
1163
|
-
return this.exec("snapshot", { path:
|
|
1503
|
+
snapshot(path10) {
|
|
1504
|
+
return this.exec("snapshot", { path: path10 });
|
|
1164
1505
|
}
|
|
1165
|
-
restore(
|
|
1166
|
-
return this.exec("restore", { path:
|
|
1506
|
+
restore(path10) {
|
|
1507
|
+
return this.exec("restore", { path: path10 });
|
|
1167
1508
|
}
|
|
1168
1509
|
/* ------------------------------ SHUTDOWN ------------------------------ */
|
|
1169
1510
|
async shutdown() {
|
|
@@ -1178,7 +1519,7 @@ var dbQueue = new DBQueue();
|
|
|
1178
1519
|
|
|
1179
1520
|
// src/ipc/server.ts
|
|
1180
1521
|
import net2 from "net";
|
|
1181
|
-
import
|
|
1522
|
+
import fs8 from "fs";
|
|
1182
1523
|
var IPCServer = class {
|
|
1183
1524
|
server;
|
|
1184
1525
|
manager;
|
|
@@ -1189,7 +1530,7 @@ var IPCServer = class {
|
|
|
1189
1530
|
}
|
|
1190
1531
|
start() {
|
|
1191
1532
|
if (!this.socketPath.startsWith("\\\\.\\")) {
|
|
1192
|
-
if (
|
|
1533
|
+
if (fs8.existsSync(this.socketPath)) fs8.unlinkSync(this.socketPath);
|
|
1193
1534
|
}
|
|
1194
1535
|
this.server = net2.createServer((socket) => {
|
|
1195
1536
|
let buffer = "";
|
|
@@ -1291,7 +1632,7 @@ var IPCServer = class {
|
|
|
1291
1632
|
if (this.server) this.server.close();
|
|
1292
1633
|
if (!this.socketPath.startsWith("\\\\.\\")) {
|
|
1293
1634
|
try {
|
|
1294
|
-
|
|
1635
|
+
fs8.unlinkSync(this.socketPath);
|
|
1295
1636
|
} catch {
|
|
1296
1637
|
}
|
|
1297
1638
|
}
|
|
@@ -1309,8 +1650,8 @@ var LioranManager = class {
|
|
|
1309
1650
|
constructor(options = {}) {
|
|
1310
1651
|
const { rootPath, encryptionKey } = options;
|
|
1311
1652
|
this.rootPath = rootPath || getDefaultRootPath();
|
|
1312
|
-
if (!
|
|
1313
|
-
|
|
1653
|
+
if (!fs9.existsSync(this.rootPath)) {
|
|
1654
|
+
fs9.mkdirSync(this.rootPath, { recursive: true });
|
|
1314
1655
|
}
|
|
1315
1656
|
if (encryptionKey) {
|
|
1316
1657
|
setEncryptionKey(encryptionKey);
|
|
@@ -1333,18 +1674,18 @@ var LioranManager = class {
|
|
|
1333
1674
|
}
|
|
1334
1675
|
}
|
|
1335
1676
|
tryAcquireLock() {
|
|
1336
|
-
const lockPath =
|
|
1677
|
+
const lockPath = path9.join(this.rootPath, ".lioran.lock");
|
|
1337
1678
|
try {
|
|
1338
|
-
this.lockFd =
|
|
1339
|
-
|
|
1679
|
+
this.lockFd = fs9.openSync(lockPath, "wx");
|
|
1680
|
+
fs9.writeSync(this.lockFd, String(process2.pid));
|
|
1340
1681
|
return true;
|
|
1341
1682
|
} catch {
|
|
1342
1683
|
try {
|
|
1343
|
-
const pid = Number(
|
|
1684
|
+
const pid = Number(fs9.readFileSync(lockPath, "utf8"));
|
|
1344
1685
|
if (!this.isProcessAlive(pid)) {
|
|
1345
|
-
|
|
1346
|
-
this.lockFd =
|
|
1347
|
-
|
|
1686
|
+
fs9.unlinkSync(lockPath);
|
|
1687
|
+
this.lockFd = fs9.openSync(lockPath, "wx");
|
|
1688
|
+
fs9.writeSync(this.lockFd, String(process2.pid));
|
|
1348
1689
|
return true;
|
|
1349
1690
|
}
|
|
1350
1691
|
} catch {
|
|
@@ -1365,8 +1706,8 @@ var LioranManager = class {
|
|
|
1365
1706
|
if (this.openDBs.has(name)) {
|
|
1366
1707
|
return this.openDBs.get(name);
|
|
1367
1708
|
}
|
|
1368
|
-
const dbPath =
|
|
1369
|
-
await
|
|
1709
|
+
const dbPath = path9.join(this.rootPath, name);
|
|
1710
|
+
await fs9.promises.mkdir(dbPath, { recursive: true });
|
|
1370
1711
|
const db = new LioranDB(dbPath, name, this);
|
|
1371
1712
|
this.openDBs.set(name, db);
|
|
1372
1713
|
return db;
|
|
@@ -1387,7 +1728,7 @@ var LioranManager = class {
|
|
|
1387
1728
|
}
|
|
1388
1729
|
}
|
|
1389
1730
|
}
|
|
1390
|
-
|
|
1731
|
+
fs9.mkdirSync(path9.dirname(snapshotPath), { recursive: true });
|
|
1391
1732
|
const tar = await import("tar");
|
|
1392
1733
|
await tar.c({
|
|
1393
1734
|
gzip: true,
|
|
@@ -1405,8 +1746,8 @@ var LioranManager = class {
|
|
|
1405
1746
|
return dbQueue.exec("restore", { path: snapshotPath });
|
|
1406
1747
|
}
|
|
1407
1748
|
await this.closeAll();
|
|
1408
|
-
|
|
1409
|
-
|
|
1749
|
+
fs9.rmSync(this.rootPath, { recursive: true, force: true });
|
|
1750
|
+
fs9.mkdirSync(this.rootPath, { recursive: true });
|
|
1410
1751
|
const tar = await import("tar");
|
|
1411
1752
|
await tar.x({
|
|
1412
1753
|
file: snapshotPath,
|
|
@@ -1431,8 +1772,8 @@ var LioranManager = class {
|
|
|
1431
1772
|
}
|
|
1432
1773
|
this.openDBs.clear();
|
|
1433
1774
|
try {
|
|
1434
|
-
if (this.lockFd)
|
|
1435
|
-
|
|
1775
|
+
if (this.lockFd) fs9.closeSync(this.lockFd);
|
|
1776
|
+
fs9.unlinkSync(path9.join(this.rootPath, ".lioran.lock"));
|
|
1436
1777
|
} catch {
|
|
1437
1778
|
}
|
|
1438
1779
|
await this.ipcServer?.close();
|