@liorandb/core 1.0.16 → 1.0.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +5 -0
- package/dist/index.js +214 -89
- package/package.json +1 -1
- package/src/core/database.ts +68 -54
- package/src/core/migration.store.ts +40 -0
- package/src/core/migration.ts +162 -0
- package/src/core/migration.types.ts +22 -0
package/dist/index.d.ts
CHANGED
|
@@ -92,10 +92,15 @@ declare class LioranDB {
|
|
|
92
92
|
private walPath;
|
|
93
93
|
private metaPath;
|
|
94
94
|
private meta;
|
|
95
|
+
private migrator;
|
|
95
96
|
private static TX_SEQ;
|
|
96
97
|
constructor(basePath: string, dbName: string, manager: LioranManager);
|
|
97
98
|
private loadMeta;
|
|
98
99
|
private saveMeta;
|
|
100
|
+
getSchemaVersion(): string;
|
|
101
|
+
setSchemaVersion(v: string): void;
|
|
102
|
+
migrate(from: string, to: string, fn: (db: LioranDB) => Promise<void>): void;
|
|
103
|
+
applyMigrations(targetVersion: string): Promise<void>;
|
|
99
104
|
writeWAL(entries: WALEntry[]): Promise<void>;
|
|
100
105
|
clearWAL(): Promise<void>;
|
|
101
106
|
private recoverFromWAL;
|
package/dist/index.js
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
// src/LioranManager.ts
|
|
2
|
-
import
|
|
3
|
-
import
|
|
2
|
+
import path7 from "path";
|
|
3
|
+
import fs7 from "fs";
|
|
4
4
|
import process2 from "process";
|
|
5
5
|
|
|
6
6
|
// src/core/database.ts
|
|
7
|
-
import
|
|
8
|
-
import
|
|
7
|
+
import path4 from "path";
|
|
8
|
+
import fs4 from "fs";
|
|
9
9
|
import { execFile } from "child_process";
|
|
10
10
|
import { promisify } from "util";
|
|
11
11
|
|
|
@@ -13,8 +13,8 @@ import { promisify } from "util";
|
|
|
13
13
|
import { ClassicLevel as ClassicLevel3 } from "classic-level";
|
|
14
14
|
|
|
15
15
|
// src/core/query.ts
|
|
16
|
-
function getByPath(obj,
|
|
17
|
-
return
|
|
16
|
+
function getByPath(obj, path8) {
|
|
17
|
+
return path8.split(".").reduce((o, p) => o ? o[p] : void 0, obj);
|
|
18
18
|
}
|
|
19
19
|
function matchDocument(doc, query) {
|
|
20
20
|
for (const key of Object.keys(query)) {
|
|
@@ -670,10 +670,134 @@ var Collection = class {
|
|
|
670
670
|
}
|
|
671
671
|
};
|
|
672
672
|
|
|
673
|
+
// src/core/migration.ts
|
|
674
|
+
import fs3 from "fs";
|
|
675
|
+
import path3 from "path";
|
|
676
|
+
import crypto3 from "crypto";
|
|
677
|
+
var LOCK_FILE = "__migration.lock";
|
|
678
|
+
var HISTORY_FILE = "__migration_history.json";
|
|
679
|
+
var MigrationEngine = class {
|
|
680
|
+
constructor(db) {
|
|
681
|
+
this.db = db;
|
|
682
|
+
}
|
|
683
|
+
migrations = /* @__PURE__ */ new Map();
|
|
684
|
+
/* ------------------------------------------------------------ */
|
|
685
|
+
/* Public API */
|
|
686
|
+
/* ------------------------------------------------------------ */
|
|
687
|
+
register(from, to, fn) {
|
|
688
|
+
const key = `${from}\u2192${to}`;
|
|
689
|
+
if (this.migrations.has(key)) {
|
|
690
|
+
throw new Error(`Duplicate migration: ${key}`);
|
|
691
|
+
}
|
|
692
|
+
this.migrations.set(key, fn);
|
|
693
|
+
}
|
|
694
|
+
async migrate(from, to, fn) {
|
|
695
|
+
this.register(from, to, fn);
|
|
696
|
+
await this.execute();
|
|
697
|
+
}
|
|
698
|
+
async upgradeToLatest() {
|
|
699
|
+
await this.execute();
|
|
700
|
+
}
|
|
701
|
+
/* ------------------------------------------------------------ */
|
|
702
|
+
/* Core Execution Logic */
|
|
703
|
+
/* ------------------------------------------------------------ */
|
|
704
|
+
async execute() {
|
|
705
|
+
let current = this.db.getSchemaVersion();
|
|
706
|
+
while (true) {
|
|
707
|
+
const next = this.findNext(current);
|
|
708
|
+
if (!next) break;
|
|
709
|
+
const fn = this.migrations.get(`${current}\u2192${next}`);
|
|
710
|
+
await this.runMigration(current, next, fn);
|
|
711
|
+
current = next;
|
|
712
|
+
}
|
|
713
|
+
}
|
|
714
|
+
findNext(current) {
|
|
715
|
+
for (const key of this.migrations.keys()) {
|
|
716
|
+
const [from, to] = key.split("\u2192");
|
|
717
|
+
if (from === current) return to;
|
|
718
|
+
}
|
|
719
|
+
return null;
|
|
720
|
+
}
|
|
721
|
+
/* ------------------------------------------------------------ */
|
|
722
|
+
/* Atomic Migration Execution */
|
|
723
|
+
/* ------------------------------------------------------------ */
|
|
724
|
+
async runMigration(from, to, fn) {
|
|
725
|
+
const current = this.db.getSchemaVersion();
|
|
726
|
+
if (current !== from) {
|
|
727
|
+
throw new Error(
|
|
728
|
+
`Schema mismatch: DB=${current}, expected=${from}`
|
|
729
|
+
);
|
|
730
|
+
}
|
|
731
|
+
const lockPath = path3.join(this.db.basePath, LOCK_FILE);
|
|
732
|
+
if (fs3.existsSync(lockPath)) {
|
|
733
|
+
throw new Error(
|
|
734
|
+
"Previous migration interrupted. Resolve manually before continuing."
|
|
735
|
+
);
|
|
736
|
+
}
|
|
737
|
+
this.acquireLock(lockPath);
|
|
738
|
+
try {
|
|
739
|
+
await this.db.transaction(async () => {
|
|
740
|
+
await fn(this.db);
|
|
741
|
+
this.writeHistory(from, to, fn);
|
|
742
|
+
this.db.setSchemaVersion(to);
|
|
743
|
+
});
|
|
744
|
+
} finally {
|
|
745
|
+
this.releaseLock(lockPath);
|
|
746
|
+
}
|
|
747
|
+
}
|
|
748
|
+
/* ------------------------------------------------------------ */
|
|
749
|
+
/* Locking */
|
|
750
|
+
/* ------------------------------------------------------------ */
|
|
751
|
+
acquireLock(file) {
|
|
752
|
+
const token = crypto3.randomBytes(16).toString("hex");
|
|
753
|
+
fs3.writeFileSync(
|
|
754
|
+
file,
|
|
755
|
+
JSON.stringify({
|
|
756
|
+
pid: process.pid,
|
|
757
|
+
token,
|
|
758
|
+
time: Date.now()
|
|
759
|
+
})
|
|
760
|
+
);
|
|
761
|
+
}
|
|
762
|
+
releaseLock(file) {
|
|
763
|
+
if (fs3.existsSync(file)) fs3.unlinkSync(file);
|
|
764
|
+
}
|
|
765
|
+
/* ------------------------------------------------------------ */
|
|
766
|
+
/* Migration History */
|
|
767
|
+
/* ------------------------------------------------------------ */
|
|
768
|
+
historyPath() {
|
|
769
|
+
return path3.join(this.db.basePath, HISTORY_FILE);
|
|
770
|
+
}
|
|
771
|
+
readHistory() {
|
|
772
|
+
if (!fs3.existsSync(this.historyPath())) return [];
|
|
773
|
+
return JSON.parse(fs3.readFileSync(this.historyPath(), "utf8"));
|
|
774
|
+
}
|
|
775
|
+
writeHistory(from, to, fn) {
|
|
776
|
+
const history = this.readHistory();
|
|
777
|
+
history.push({
|
|
778
|
+
from,
|
|
779
|
+
to,
|
|
780
|
+
checksum: this.hash(fn.toString()),
|
|
781
|
+
appliedAt: Date.now()
|
|
782
|
+
});
|
|
783
|
+
fs3.writeFileSync(this.historyPath(), JSON.stringify(history, null, 2));
|
|
784
|
+
}
|
|
785
|
+
hash(data) {
|
|
786
|
+
return crypto3.createHash("sha256").update(data).digest("hex");
|
|
787
|
+
}
|
|
788
|
+
/* ------------------------------------------------------------ */
|
|
789
|
+
/* Diagnostics */
|
|
790
|
+
/* ------------------------------------------------------------ */
|
|
791
|
+
getHistory() {
|
|
792
|
+
return this.readHistory();
|
|
793
|
+
}
|
|
794
|
+
};
|
|
795
|
+
|
|
673
796
|
// src/core/database.ts
|
|
674
797
|
var exec = promisify(execFile);
|
|
675
798
|
var META_FILE = "__db_meta.json";
|
|
676
799
|
var META_VERSION = 1;
|
|
800
|
+
var DEFAULT_SCHEMA_VERSION = "v1";
|
|
677
801
|
var DBTransactionContext = class {
|
|
678
802
|
constructor(db, txId) {
|
|
679
803
|
this.db = db;
|
|
@@ -710,33 +834,60 @@ var LioranDB = class _LioranDB {
|
|
|
710
834
|
walPath;
|
|
711
835
|
metaPath;
|
|
712
836
|
meta;
|
|
837
|
+
migrator;
|
|
713
838
|
static TX_SEQ = 0;
|
|
714
839
|
constructor(basePath, dbName, manager) {
|
|
715
840
|
this.basePath = basePath;
|
|
716
841
|
this.dbName = dbName;
|
|
717
842
|
this.manager = manager;
|
|
718
843
|
this.collections = /* @__PURE__ */ new Map();
|
|
719
|
-
this.walPath =
|
|
720
|
-
this.metaPath =
|
|
721
|
-
|
|
844
|
+
this.walPath = path4.join(basePath, "__tx_wal.log");
|
|
845
|
+
this.metaPath = path4.join(basePath, META_FILE);
|
|
846
|
+
fs4.mkdirSync(basePath, { recursive: true });
|
|
722
847
|
this.loadMeta();
|
|
848
|
+
this.migrator = new MigrationEngine(this);
|
|
723
849
|
this.recoverFromWAL().catch(console.error);
|
|
724
850
|
}
|
|
725
851
|
/* ------------------------- META ------------------------- */
|
|
726
852
|
loadMeta() {
|
|
727
|
-
if (!
|
|
728
|
-
this.meta = {
|
|
853
|
+
if (!fs4.existsSync(this.metaPath)) {
|
|
854
|
+
this.meta = {
|
|
855
|
+
version: META_VERSION,
|
|
856
|
+
indexes: {},
|
|
857
|
+
schemaVersion: DEFAULT_SCHEMA_VERSION
|
|
858
|
+
};
|
|
729
859
|
this.saveMeta();
|
|
730
860
|
return;
|
|
731
861
|
}
|
|
732
|
-
this.meta = JSON.parse(
|
|
862
|
+
this.meta = JSON.parse(fs4.readFileSync(this.metaPath, "utf8"));
|
|
863
|
+
if (!this.meta.schemaVersion) {
|
|
864
|
+
this.meta.schemaVersion = DEFAULT_SCHEMA_VERSION;
|
|
865
|
+
this.saveMeta();
|
|
866
|
+
}
|
|
733
867
|
}
|
|
734
868
|
saveMeta() {
|
|
735
|
-
|
|
869
|
+
fs4.writeFileSync(this.metaPath, JSON.stringify(this.meta, null, 2));
|
|
870
|
+
}
|
|
871
|
+
getSchemaVersion() {
|
|
872
|
+
return this.meta.schemaVersion;
|
|
873
|
+
}
|
|
874
|
+
setSchemaVersion(v) {
|
|
875
|
+
this.meta.schemaVersion = v;
|
|
876
|
+
this.saveMeta();
|
|
877
|
+
}
|
|
878
|
+
/* ------------------------- MIGRATION API ------------------------- */
|
|
879
|
+
migrate(from, to, fn) {
|
|
880
|
+
this.migrator.register(from, to, async (db) => {
|
|
881
|
+
await fn(db);
|
|
882
|
+
db.setSchemaVersion(to);
|
|
883
|
+
});
|
|
884
|
+
}
|
|
885
|
+
async applyMigrations(targetVersion) {
|
|
886
|
+
await this.migrator.upgradeToLatest();
|
|
736
887
|
}
|
|
737
888
|
/* ------------------------- WAL ------------------------- */
|
|
738
889
|
async writeWAL(entries) {
|
|
739
|
-
const fd = await
|
|
890
|
+
const fd = await fs4.promises.open(this.walPath, "a");
|
|
740
891
|
for (const e of entries) {
|
|
741
892
|
await fd.write(JSON.stringify(e) + "\n");
|
|
742
893
|
}
|
|
@@ -745,13 +896,13 @@ var LioranDB = class _LioranDB {
|
|
|
745
896
|
}
|
|
746
897
|
async clearWAL() {
|
|
747
898
|
try {
|
|
748
|
-
await
|
|
899
|
+
await fs4.promises.unlink(this.walPath);
|
|
749
900
|
} catch {
|
|
750
901
|
}
|
|
751
902
|
}
|
|
752
903
|
async recoverFromWAL() {
|
|
753
|
-
if (!
|
|
754
|
-
const raw = await
|
|
904
|
+
if (!fs4.existsSync(this.walPath)) return;
|
|
905
|
+
const raw = await fs4.promises.readFile(this.walPath, "utf8");
|
|
755
906
|
const committed = /* @__PURE__ */ new Set();
|
|
756
907
|
const applied = /* @__PURE__ */ new Set();
|
|
757
908
|
const ops = /* @__PURE__ */ new Map();
|
|
@@ -785,8 +936,8 @@ var LioranDB = class _LioranDB {
|
|
|
785
936
|
if (schema) col2.setSchema(schema);
|
|
786
937
|
return col2;
|
|
787
938
|
}
|
|
788
|
-
const colPath =
|
|
789
|
-
|
|
939
|
+
const colPath = path4.join(this.basePath, name);
|
|
940
|
+
fs4.mkdirSync(colPath, { recursive: true });
|
|
790
941
|
const col = new Collection(colPath, schema);
|
|
791
942
|
const metas = this.meta.indexes[name] ?? [];
|
|
792
943
|
for (const m of metas) {
|
|
@@ -801,9 +952,15 @@ var LioranDB = class _LioranDB {
|
|
|
801
952
|
const existing = this.meta.indexes[collection]?.find((i) => i.field === field);
|
|
802
953
|
if (existing) return;
|
|
803
954
|
const index = new Index(col.dir, field, options);
|
|
804
|
-
for await (const [, enc] of col.db.iterator()) {
|
|
805
|
-
|
|
806
|
-
|
|
955
|
+
for await (const [key, enc] of col.db.iterator()) {
|
|
956
|
+
if (!enc) continue;
|
|
957
|
+
try {
|
|
958
|
+
const doc = decryptData2(enc);
|
|
959
|
+
await index.insert(doc);
|
|
960
|
+
} catch (err) {
|
|
961
|
+
const errorMessage = err instanceof Error ? err.message : String(err);
|
|
962
|
+
console.warn(`Could not decrypt document ${key} during index build: ${errorMessage}`);
|
|
963
|
+
}
|
|
807
964
|
}
|
|
808
965
|
col.registerIndex(index);
|
|
809
966
|
if (!this.meta.indexes[collection]) {
|
|
@@ -812,41 +969,6 @@ var LioranDB = class _LioranDB {
|
|
|
812
969
|
this.meta.indexes[collection].push({ field, options });
|
|
813
970
|
this.saveMeta();
|
|
814
971
|
}
|
|
815
|
-
/* ------------------------- SNAPSHOT ENGINE ------------------------- */
|
|
816
|
-
// async snapshot(snapshotPath: string) {
|
|
817
|
-
// await this.clearWAL();
|
|
818
|
-
// for (const col of this.collections.values()) {
|
|
819
|
-
// await col.close();
|
|
820
|
-
// }
|
|
821
|
-
// fs.mkdirSync(path.dirname(snapshotPath), { recursive: true });
|
|
822
|
-
// await exec("tar", [
|
|
823
|
-
// "-czf",
|
|
824
|
-
// snapshotPath,
|
|
825
|
-
// "-C",
|
|
826
|
-
// this.basePath,
|
|
827
|
-
// "."
|
|
828
|
-
// ]);
|
|
829
|
-
// }
|
|
830
|
-
// async restore(snapshotPath: string) {
|
|
831
|
-
// if (!fs.existsSync(snapshotPath)) {
|
|
832
|
-
// throw new Error("Snapshot file does not exist");
|
|
833
|
-
// }
|
|
834
|
-
// await this.close();
|
|
835
|
-
// const tmp = this.basePath + ".restore";
|
|
836
|
-
// await fs.promises.rm(tmp, { recursive: true, force: true });
|
|
837
|
-
// await fs.promises.mkdir(tmp, { recursive: true });
|
|
838
|
-
// await exec("tar", [
|
|
839
|
-
// "-xzf",
|
|
840
|
-
// snapshotPath,
|
|
841
|
-
// "-C",
|
|
842
|
-
// tmp
|
|
843
|
-
// ]);
|
|
844
|
-
// await fs.promises.rm(this.basePath, { recursive: true, force: true });
|
|
845
|
-
// await fs.promises.rename(tmp, this.basePath);
|
|
846
|
-
// this.collections.clear();
|
|
847
|
-
// this.loadMeta();
|
|
848
|
-
// await this.recoverFromWAL();
|
|
849
|
-
// }
|
|
850
972
|
/* ------------------------- COMPACTION ------------------------- */
|
|
851
973
|
async compactCollection(name) {
|
|
852
974
|
await this.clearWAL();
|
|
@@ -878,18 +1000,21 @@ var LioranDB = class _LioranDB {
|
|
|
878
1000
|
this.collections.clear();
|
|
879
1001
|
}
|
|
880
1002
|
};
|
|
1003
|
+
function decryptData2(enc) {
|
|
1004
|
+
throw new Error("Function not implemented.");
|
|
1005
|
+
}
|
|
881
1006
|
|
|
882
1007
|
// src/utils/rootpath.ts
|
|
883
1008
|
import os2 from "os";
|
|
884
|
-
import
|
|
885
|
-
import
|
|
1009
|
+
import path5 from "path";
|
|
1010
|
+
import fs5 from "fs";
|
|
886
1011
|
function getDefaultRootPath() {
|
|
887
1012
|
let dbPath = process.env.LIORANDB_PATH;
|
|
888
1013
|
if (!dbPath) {
|
|
889
1014
|
const homeDir = os2.homedir();
|
|
890
|
-
dbPath =
|
|
891
|
-
if (!
|
|
892
|
-
|
|
1015
|
+
dbPath = path5.join(homeDir, "LioranDB", "db");
|
|
1016
|
+
if (!fs5.existsSync(dbPath)) {
|
|
1017
|
+
fs5.mkdirSync(dbPath, { recursive: true });
|
|
893
1018
|
}
|
|
894
1019
|
process.env.LIORANDB_PATH = dbPath;
|
|
895
1020
|
}
|
|
@@ -904,24 +1029,24 @@ import net from "net";
|
|
|
904
1029
|
|
|
905
1030
|
// src/ipc/socketPath.ts
|
|
906
1031
|
import os3 from "os";
|
|
907
|
-
import
|
|
1032
|
+
import path6 from "path";
|
|
908
1033
|
function getIPCSocketPath(rootPath) {
|
|
909
1034
|
if (os3.platform() === "win32") {
|
|
910
1035
|
return `\\\\.\\pipe\\liorandb_${rootPath.replace(/[:\\\/]/g, "_")}`;
|
|
911
1036
|
}
|
|
912
|
-
return
|
|
1037
|
+
return path6.join(rootPath, ".lioran.sock");
|
|
913
1038
|
}
|
|
914
1039
|
|
|
915
1040
|
// src/ipc/client.ts
|
|
916
1041
|
function delay(ms) {
|
|
917
1042
|
return new Promise((r) => setTimeout(r, ms));
|
|
918
1043
|
}
|
|
919
|
-
async function connectWithRetry(
|
|
1044
|
+
async function connectWithRetry(path8) {
|
|
920
1045
|
let attempt = 0;
|
|
921
1046
|
while (true) {
|
|
922
1047
|
try {
|
|
923
1048
|
return await new Promise((resolve, reject) => {
|
|
924
|
-
const socket = net.connect(
|
|
1049
|
+
const socket = net.connect(path8, () => resolve(socket));
|
|
925
1050
|
socket.once("error", reject);
|
|
926
1051
|
});
|
|
927
1052
|
} catch (err) {
|
|
@@ -1004,11 +1129,11 @@ var DBQueue = class {
|
|
|
1004
1129
|
return this.exec("compact:all", {});
|
|
1005
1130
|
}
|
|
1006
1131
|
/* ----------------------------- SNAPSHOT API ----------------------------- */
|
|
1007
|
-
snapshot(
|
|
1008
|
-
return this.exec("snapshot", { path:
|
|
1132
|
+
snapshot(path8) {
|
|
1133
|
+
return this.exec("snapshot", { path: path8 });
|
|
1009
1134
|
}
|
|
1010
|
-
restore(
|
|
1011
|
-
return this.exec("restore", { path:
|
|
1135
|
+
restore(path8) {
|
|
1136
|
+
return this.exec("restore", { path: path8 });
|
|
1012
1137
|
}
|
|
1013
1138
|
/* ------------------------------ SHUTDOWN ------------------------------ */
|
|
1014
1139
|
async shutdown() {
|
|
@@ -1023,7 +1148,7 @@ var dbQueue = new DBQueue();
|
|
|
1023
1148
|
|
|
1024
1149
|
// src/ipc/server.ts
|
|
1025
1150
|
import net2 from "net";
|
|
1026
|
-
import
|
|
1151
|
+
import fs6 from "fs";
|
|
1027
1152
|
var IPCServer = class {
|
|
1028
1153
|
server;
|
|
1029
1154
|
manager;
|
|
@@ -1034,7 +1159,7 @@ var IPCServer = class {
|
|
|
1034
1159
|
}
|
|
1035
1160
|
start() {
|
|
1036
1161
|
if (!this.socketPath.startsWith("\\\\.\\")) {
|
|
1037
|
-
if (
|
|
1162
|
+
if (fs6.existsSync(this.socketPath)) fs6.unlinkSync(this.socketPath);
|
|
1038
1163
|
}
|
|
1039
1164
|
this.server = net2.createServer((socket) => {
|
|
1040
1165
|
let buffer = "";
|
|
@@ -1136,7 +1261,7 @@ var IPCServer = class {
|
|
|
1136
1261
|
if (this.server) this.server.close();
|
|
1137
1262
|
if (!this.socketPath.startsWith("\\\\.\\")) {
|
|
1138
1263
|
try {
|
|
1139
|
-
|
|
1264
|
+
fs6.unlinkSync(this.socketPath);
|
|
1140
1265
|
} catch {
|
|
1141
1266
|
}
|
|
1142
1267
|
}
|
|
@@ -1154,8 +1279,8 @@ var LioranManager = class {
|
|
|
1154
1279
|
constructor(options = {}) {
|
|
1155
1280
|
const { rootPath, encryptionKey } = options;
|
|
1156
1281
|
this.rootPath = rootPath || getDefaultRootPath();
|
|
1157
|
-
if (!
|
|
1158
|
-
|
|
1282
|
+
if (!fs7.existsSync(this.rootPath)) {
|
|
1283
|
+
fs7.mkdirSync(this.rootPath, { recursive: true });
|
|
1159
1284
|
}
|
|
1160
1285
|
if (encryptionKey) {
|
|
1161
1286
|
setEncryptionKey(encryptionKey);
|
|
@@ -1178,18 +1303,18 @@ var LioranManager = class {
|
|
|
1178
1303
|
}
|
|
1179
1304
|
}
|
|
1180
1305
|
tryAcquireLock() {
|
|
1181
|
-
const lockPath =
|
|
1306
|
+
const lockPath = path7.join(this.rootPath, ".lioran.lock");
|
|
1182
1307
|
try {
|
|
1183
|
-
this.lockFd =
|
|
1184
|
-
|
|
1308
|
+
this.lockFd = fs7.openSync(lockPath, "wx");
|
|
1309
|
+
fs7.writeSync(this.lockFd, String(process2.pid));
|
|
1185
1310
|
return true;
|
|
1186
1311
|
} catch {
|
|
1187
1312
|
try {
|
|
1188
|
-
const pid = Number(
|
|
1313
|
+
const pid = Number(fs7.readFileSync(lockPath, "utf8"));
|
|
1189
1314
|
if (!this.isProcessAlive(pid)) {
|
|
1190
|
-
|
|
1191
|
-
this.lockFd =
|
|
1192
|
-
|
|
1315
|
+
fs7.unlinkSync(lockPath);
|
|
1316
|
+
this.lockFd = fs7.openSync(lockPath, "wx");
|
|
1317
|
+
fs7.writeSync(this.lockFd, String(process2.pid));
|
|
1193
1318
|
return true;
|
|
1194
1319
|
}
|
|
1195
1320
|
} catch {
|
|
@@ -1210,8 +1335,8 @@ var LioranManager = class {
|
|
|
1210
1335
|
if (this.openDBs.has(name)) {
|
|
1211
1336
|
return this.openDBs.get(name);
|
|
1212
1337
|
}
|
|
1213
|
-
const dbPath =
|
|
1214
|
-
await
|
|
1338
|
+
const dbPath = path7.join(this.rootPath, name);
|
|
1339
|
+
await fs7.promises.mkdir(dbPath, { recursive: true });
|
|
1215
1340
|
const db = new LioranDB(dbPath, name, this);
|
|
1216
1341
|
this.openDBs.set(name, db);
|
|
1217
1342
|
return db;
|
|
@@ -1232,7 +1357,7 @@ var LioranManager = class {
|
|
|
1232
1357
|
}
|
|
1233
1358
|
}
|
|
1234
1359
|
}
|
|
1235
|
-
|
|
1360
|
+
fs7.mkdirSync(path7.dirname(snapshotPath), { recursive: true });
|
|
1236
1361
|
const tar = await import("tar");
|
|
1237
1362
|
await tar.c({
|
|
1238
1363
|
gzip: true,
|
|
@@ -1250,8 +1375,8 @@ var LioranManager = class {
|
|
|
1250
1375
|
return dbQueue.exec("restore", { path: snapshotPath });
|
|
1251
1376
|
}
|
|
1252
1377
|
await this.closeAll();
|
|
1253
|
-
|
|
1254
|
-
|
|
1378
|
+
fs7.rmSync(this.rootPath, { recursive: true, force: true });
|
|
1379
|
+
fs7.mkdirSync(this.rootPath, { recursive: true });
|
|
1255
1380
|
const tar = await import("tar");
|
|
1256
1381
|
await tar.x({
|
|
1257
1382
|
file: snapshotPath,
|
|
@@ -1276,8 +1401,8 @@ var LioranManager = class {
|
|
|
1276
1401
|
}
|
|
1277
1402
|
this.openDBs.clear();
|
|
1278
1403
|
try {
|
|
1279
|
-
if (this.lockFd)
|
|
1280
|
-
|
|
1404
|
+
if (this.lockFd) fs7.closeSync(this.lockFd);
|
|
1405
|
+
fs7.unlinkSync(path7.join(this.rootPath, ".lioran.lock"));
|
|
1281
1406
|
} catch {
|
|
1282
1407
|
}
|
|
1283
1408
|
await this.ipcServer?.close();
|
package/package.json
CHANGED
package/src/core/database.ts
CHANGED
|
@@ -4,6 +4,7 @@ import { execFile } from "child_process";
|
|
|
4
4
|
import { promisify } from "util";
|
|
5
5
|
import { Collection } from "./collection.js";
|
|
6
6
|
import { Index, IndexOptions } from "./index.js";
|
|
7
|
+
import { MigrationEngine } from "./migration.js";
|
|
7
8
|
import type { LioranManager } from "../LioranManager.js";
|
|
8
9
|
import type { ZodSchema } from "zod";
|
|
9
10
|
|
|
@@ -24,10 +25,12 @@ type IndexMeta = {
|
|
|
24
25
|
type DBMeta = {
|
|
25
26
|
version: number;
|
|
26
27
|
indexes: Record<string, IndexMeta[]>;
|
|
28
|
+
schemaVersion: string;
|
|
27
29
|
};
|
|
28
30
|
|
|
29
31
|
const META_FILE = "__db_meta.json";
|
|
30
32
|
const META_VERSION = 1;
|
|
33
|
+
const DEFAULT_SCHEMA_VERSION = "v1";
|
|
31
34
|
|
|
32
35
|
/* ---------------------- TRANSACTION CONTEXT ---------------------- */
|
|
33
36
|
|
|
@@ -37,7 +40,7 @@ class DBTransactionContext {
|
|
|
37
40
|
constructor(
|
|
38
41
|
private db: LioranDB,
|
|
39
42
|
public readonly txId: number
|
|
40
|
-
) {}
|
|
43
|
+
) { }
|
|
41
44
|
|
|
42
45
|
collection(name: string) {
|
|
43
46
|
return new Proxy({}, {
|
|
@@ -70,10 +73,13 @@ export class LioranDB {
|
|
|
70
73
|
dbName: string;
|
|
71
74
|
manager: LioranManager;
|
|
72
75
|
collections: Map<string, Collection>;
|
|
76
|
+
|
|
73
77
|
private walPath: string;
|
|
74
78
|
private metaPath: string;
|
|
75
79
|
private meta!: DBMeta;
|
|
76
80
|
|
|
81
|
+
private migrator: MigrationEngine;
|
|
82
|
+
|
|
77
83
|
private static TX_SEQ = 0;
|
|
78
84
|
|
|
79
85
|
constructor(basePath: string, dbName: string, manager: LioranManager) {
|
|
@@ -81,12 +87,15 @@ export class LioranDB {
|
|
|
81
87
|
this.dbName = dbName;
|
|
82
88
|
this.manager = manager;
|
|
83
89
|
this.collections = new Map();
|
|
90
|
+
|
|
84
91
|
this.walPath = path.join(basePath, "__tx_wal.log");
|
|
85
92
|
this.metaPath = path.join(basePath, META_FILE);
|
|
86
93
|
|
|
87
94
|
fs.mkdirSync(basePath, { recursive: true });
|
|
88
95
|
|
|
89
96
|
this.loadMeta();
|
|
97
|
+
this.migrator = new MigrationEngine(this);
|
|
98
|
+
|
|
90
99
|
this.recoverFromWAL().catch(console.error);
|
|
91
100
|
}
|
|
92
101
|
|
|
@@ -94,18 +103,49 @@ export class LioranDB {
|
|
|
94
103
|
|
|
95
104
|
private loadMeta() {
|
|
96
105
|
if (!fs.existsSync(this.metaPath)) {
|
|
97
|
-
this.meta = {
|
|
106
|
+
this.meta = {
|
|
107
|
+
version: META_VERSION,
|
|
108
|
+
indexes: {},
|
|
109
|
+
schemaVersion: DEFAULT_SCHEMA_VERSION
|
|
110
|
+
};
|
|
98
111
|
this.saveMeta();
|
|
99
112
|
return;
|
|
100
113
|
}
|
|
101
114
|
|
|
102
115
|
this.meta = JSON.parse(fs.readFileSync(this.metaPath, "utf8"));
|
|
116
|
+
|
|
117
|
+
if (!this.meta.schemaVersion) {
|
|
118
|
+
this.meta.schemaVersion = DEFAULT_SCHEMA_VERSION;
|
|
119
|
+
this.saveMeta();
|
|
120
|
+
}
|
|
103
121
|
}
|
|
104
122
|
|
|
105
123
|
private saveMeta() {
|
|
106
124
|
fs.writeFileSync(this.metaPath, JSON.stringify(this.meta, null, 2));
|
|
107
125
|
}
|
|
108
126
|
|
|
127
|
+
getSchemaVersion(): string {
|
|
128
|
+
return this.meta.schemaVersion;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
setSchemaVersion(v: string) {
|
|
132
|
+
this.meta.schemaVersion = v;
|
|
133
|
+
this.saveMeta();
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/* ------------------------- MIGRATION API ------------------------- */
|
|
137
|
+
|
|
138
|
+
migrate(from: string, to: string, fn: (db: LioranDB) => Promise<void>) {
|
|
139
|
+
this.migrator.register(from, to, async db => {
|
|
140
|
+
await fn(db);
|
|
141
|
+
db.setSchemaVersion(to);
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
async applyMigrations(targetVersion: string) {
|
|
146
|
+
await this.migrator.upgradeToLatest();
|
|
147
|
+
}
|
|
148
|
+
|
|
109
149
|
/* ------------------------- WAL ------------------------- */
|
|
110
150
|
|
|
111
151
|
async writeWAL(entries: WALEntry[]) {
|
|
@@ -118,7 +158,7 @@ export class LioranDB {
|
|
|
118
158
|
}
|
|
119
159
|
|
|
120
160
|
async clearWAL() {
|
|
121
|
-
try { await fs.promises.unlink(this.walPath); } catch {}
|
|
161
|
+
try { await fs.promises.unlink(this.walPath); } catch { }
|
|
122
162
|
}
|
|
123
163
|
|
|
124
164
|
private async recoverFromWAL() {
|
|
@@ -196,9 +236,26 @@ export class LioranDB {
|
|
|
196
236
|
|
|
197
237
|
const index = new Index(col.dir, field, options);
|
|
198
238
|
|
|
199
|
-
for await (const [, enc] of col.db.iterator()) {
|
|
200
|
-
|
|
201
|
-
|
|
239
|
+
// for await (const [, enc] of col.db.iterator()) {
|
|
240
|
+
// // const doc = JSON.parse(
|
|
241
|
+
// // Buffer.from(enc, "base64").subarray(32).toString("utf8")
|
|
242
|
+
// // );
|
|
243
|
+
// const payload = Buffer.from(enc, "utf8").subarray(32);
|
|
244
|
+
// const doc = JSON.parse(payload.toString("utf8"));
|
|
245
|
+
// await index.insert(doc);
|
|
246
|
+
// }
|
|
247
|
+
|
|
248
|
+
for await (const [key, enc] of col.db.iterator()) {
|
|
249
|
+
if (!enc) continue;
|
|
250
|
+
|
|
251
|
+
try {
|
|
252
|
+
const doc = decryptData(enc); // ← this does base64 → AES-GCM → JSON
|
|
253
|
+
await index.insert(doc);
|
|
254
|
+
} catch (err) {
|
|
255
|
+
const errorMessage = err instanceof Error ? err.message : String(err);
|
|
256
|
+
console.warn(`Could not decrypt document ${key} during index build: ${errorMessage}`);
|
|
257
|
+
// You can continue, or collect bad keys for later inspection
|
|
258
|
+
}
|
|
202
259
|
}
|
|
203
260
|
|
|
204
261
|
col.registerIndex(index);
|
|
@@ -211,53 +268,6 @@ export class LioranDB {
|
|
|
211
268
|
this.saveMeta();
|
|
212
269
|
}
|
|
213
270
|
|
|
214
|
-
/* ------------------------- SNAPSHOT ENGINE ------------------------- */
|
|
215
|
-
|
|
216
|
-
// async snapshot(snapshotPath: string) {
|
|
217
|
-
// await this.clearWAL();
|
|
218
|
-
|
|
219
|
-
// for (const col of this.collections.values()) {
|
|
220
|
-
// await col.close();
|
|
221
|
-
// }
|
|
222
|
-
|
|
223
|
-
// fs.mkdirSync(path.dirname(snapshotPath), { recursive: true });
|
|
224
|
-
|
|
225
|
-
// await exec("tar", [
|
|
226
|
-
// "-czf",
|
|
227
|
-
// snapshotPath,
|
|
228
|
-
// "-C",
|
|
229
|
-
// this.basePath,
|
|
230
|
-
// "."
|
|
231
|
-
// ]);
|
|
232
|
-
// }
|
|
233
|
-
|
|
234
|
-
// async restore(snapshotPath: string) {
|
|
235
|
-
// if (!fs.existsSync(snapshotPath)) {
|
|
236
|
-
// throw new Error("Snapshot file does not exist");
|
|
237
|
-
// }
|
|
238
|
-
|
|
239
|
-
// await this.close();
|
|
240
|
-
|
|
241
|
-
// const tmp = this.basePath + ".restore";
|
|
242
|
-
|
|
243
|
-
// await fs.promises.rm(tmp, { recursive: true, force: true });
|
|
244
|
-
// await fs.promises.mkdir(tmp, { recursive: true });
|
|
245
|
-
|
|
246
|
-
// await exec("tar", [
|
|
247
|
-
// "-xzf",
|
|
248
|
-
// snapshotPath,
|
|
249
|
-
// "-C",
|
|
250
|
-
// tmp
|
|
251
|
-
// ]);
|
|
252
|
-
|
|
253
|
-
// await fs.promises.rm(this.basePath, { recursive: true, force: true });
|
|
254
|
-
// await fs.promises.rename(tmp, this.basePath);
|
|
255
|
-
|
|
256
|
-
// this.collections.clear();
|
|
257
|
-
// this.loadMeta();
|
|
258
|
-
// await this.recoverFromWAL();
|
|
259
|
-
// }
|
|
260
|
-
|
|
261
271
|
/* ------------------------- COMPACTION ------------------------- */
|
|
262
272
|
|
|
263
273
|
async compactCollection(name: string) {
|
|
@@ -287,8 +297,12 @@ export class LioranDB {
|
|
|
287
297
|
|
|
288
298
|
async close(): Promise<void> {
|
|
289
299
|
for (const col of this.collections.values()) {
|
|
290
|
-
try { await col.close(); } catch {}
|
|
300
|
+
try { await col.close(); } catch { }
|
|
291
301
|
}
|
|
292
302
|
this.collections.clear();
|
|
293
303
|
}
|
|
294
304
|
}
|
|
305
|
+
|
|
306
|
+
function decryptData(enc: string) {
|
|
307
|
+
throw new Error("Function not implemented.");
|
|
308
|
+
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { MigrationMeta } from "./migration.types";
|
|
2
|
+
import type { LioranDB } from "./database.js";
|
|
3
|
+
|
|
4
|
+
const MIGRATION_COLLECTION = "__migrations__";
|
|
5
|
+
const MIGRATION_KEY = "__migration_meta__";
|
|
6
|
+
|
|
7
|
+
export class MigrationStore {
|
|
8
|
+
constructor(private db: LioranDB) {}
|
|
9
|
+
|
|
10
|
+
async get(): Promise<MigrationMeta> {
|
|
11
|
+
const col = this.db.collection<MigrationMeta>(MIGRATION_COLLECTION);
|
|
12
|
+
const meta = await col.findOne((d: MigrationMeta) => d.id === MIGRATION_KEY);
|
|
13
|
+
return (
|
|
14
|
+
meta ?? {
|
|
15
|
+
currentVersion: "v1",
|
|
16
|
+
history: []
|
|
17
|
+
}
|
|
18
|
+
);
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
async set(meta: MigrationMeta): Promise<void> {
|
|
22
|
+
const col = this.db.collection<MigrationMeta>(MIGRATION_COLLECTION);
|
|
23
|
+
const existing = await col.findOne((d: MigrationMeta) => d.id === MIGRATION_KEY);
|
|
24
|
+
if (existing) {
|
|
25
|
+
await col.updateOne(existing._id, { ...meta, id: MIGRATION_KEY });
|
|
26
|
+
} else {
|
|
27
|
+
await col.insertOne({ ...meta, id: MIGRATION_KEY });
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
async updateVersion(version: string): Promise<void> {
|
|
32
|
+
const meta = await this.get();
|
|
33
|
+
meta.currentVersion = version;
|
|
34
|
+
meta.history.push({
|
|
35
|
+
version,
|
|
36
|
+
appliedAt: Date.now()
|
|
37
|
+
});
|
|
38
|
+
await this.set(meta);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
import fs from "fs";
|
|
2
|
+
import path from "path";
|
|
3
|
+
import crypto from "crypto";
|
|
4
|
+
import type { LioranDB } from "./database.js";
|
|
5
|
+
|
|
6
|
+
export type MigrationFn = (db: LioranDB) => Promise<void>;
|
|
7
|
+
|
|
8
|
+
type MigrationRecord = {
|
|
9
|
+
from: string;
|
|
10
|
+
to: string;
|
|
11
|
+
checksum: string;
|
|
12
|
+
appliedAt: number;
|
|
13
|
+
};
|
|
14
|
+
|
|
15
|
+
const LOCK_FILE = "__migration.lock";
|
|
16
|
+
const HISTORY_FILE = "__migration_history.json";
|
|
17
|
+
|
|
18
|
+
export class MigrationEngine {
|
|
19
|
+
private migrations = new Map<string, MigrationFn>();
|
|
20
|
+
|
|
21
|
+
constructor(private db: LioranDB) {}
|
|
22
|
+
|
|
23
|
+
/* ------------------------------------------------------------ */
|
|
24
|
+
/* Public API */
|
|
25
|
+
/* ------------------------------------------------------------ */
|
|
26
|
+
|
|
27
|
+
register(from: string, to: string, fn: MigrationFn) {
|
|
28
|
+
const key = `${from}→${to}`;
|
|
29
|
+
|
|
30
|
+
if (this.migrations.has(key)) {
|
|
31
|
+
throw new Error(`Duplicate migration: ${key}`);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
this.migrations.set(key, fn);
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
async migrate(from: string, to: string, fn: MigrationFn) {
|
|
38
|
+
this.register(from, to, fn);
|
|
39
|
+
await this.execute();
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
async upgradeToLatest() {
|
|
43
|
+
await this.execute();
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/* ------------------------------------------------------------ */
|
|
47
|
+
/* Core Execution Logic */
|
|
48
|
+
/* ------------------------------------------------------------ */
|
|
49
|
+
|
|
50
|
+
private async execute() {
|
|
51
|
+
let current = this.db.getSchemaVersion();
|
|
52
|
+
|
|
53
|
+
while (true) {
|
|
54
|
+
const next = this.findNext(current);
|
|
55
|
+
if (!next) break;
|
|
56
|
+
|
|
57
|
+
const fn = this.migrations.get(`${current}→${next}`)!;
|
|
58
|
+
await this.runMigration(current, next, fn);
|
|
59
|
+
current = next;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
private findNext(current: string): string | null {
|
|
64
|
+
for (const key of this.migrations.keys()) {
|
|
65
|
+
const [from, to] = key.split("→");
|
|
66
|
+
if (from === current) return to;
|
|
67
|
+
}
|
|
68
|
+
return null;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/* ------------------------------------------------------------ */
|
|
72
|
+
/* Atomic Migration Execution */
|
|
73
|
+
/* ------------------------------------------------------------ */
|
|
74
|
+
|
|
75
|
+
private async runMigration(from: string, to: string, fn: MigrationFn) {
|
|
76
|
+
const current = this.db.getSchemaVersion();
|
|
77
|
+
if (current !== from) {
|
|
78
|
+
throw new Error(
|
|
79
|
+
`Schema mismatch: DB=${current}, expected=${from}`
|
|
80
|
+
);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
const lockPath = path.join(this.db.basePath, LOCK_FILE);
|
|
84
|
+
|
|
85
|
+
if (fs.existsSync(lockPath)) {
|
|
86
|
+
throw new Error(
|
|
87
|
+
"Previous migration interrupted. Resolve manually before continuing."
|
|
88
|
+
);
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
this.acquireLock(lockPath);
|
|
92
|
+
|
|
93
|
+
try {
|
|
94
|
+
await this.db.transaction(async () => {
|
|
95
|
+
await fn(this.db);
|
|
96
|
+
this.writeHistory(from, to, fn);
|
|
97
|
+
this.db.setSchemaVersion(to);
|
|
98
|
+
});
|
|
99
|
+
} finally {
|
|
100
|
+
this.releaseLock(lockPath);
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/* ------------------------------------------------------------ */
|
|
105
|
+
/* Locking */
|
|
106
|
+
/* ------------------------------------------------------------ */
|
|
107
|
+
|
|
108
|
+
private acquireLock(file: string) {
|
|
109
|
+
const token = crypto.randomBytes(16).toString("hex");
|
|
110
|
+
|
|
111
|
+
fs.writeFileSync(
|
|
112
|
+
file,
|
|
113
|
+
JSON.stringify({
|
|
114
|
+
pid: process.pid,
|
|
115
|
+
token,
|
|
116
|
+
time: Date.now(),
|
|
117
|
+
})
|
|
118
|
+
);
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
private releaseLock(file: string) {
|
|
122
|
+
if (fs.existsSync(file)) fs.unlinkSync(file);
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/* ------------------------------------------------------------ */
|
|
126
|
+
/* Migration History */
|
|
127
|
+
/* ------------------------------------------------------------ */
|
|
128
|
+
|
|
129
|
+
private historyPath() {
|
|
130
|
+
return path.join(this.db.basePath, HISTORY_FILE);
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
private readHistory(): MigrationRecord[] {
|
|
134
|
+
if (!fs.existsSync(this.historyPath())) return [];
|
|
135
|
+
return JSON.parse(fs.readFileSync(this.historyPath(), "utf8"));
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
private writeHistory(from: string, to: string, fn: MigrationFn) {
|
|
139
|
+
const history = this.readHistory();
|
|
140
|
+
|
|
141
|
+
history.push({
|
|
142
|
+
from,
|
|
143
|
+
to,
|
|
144
|
+
checksum: this.hash(fn.toString()),
|
|
145
|
+
appliedAt: Date.now(),
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
fs.writeFileSync(this.historyPath(), JSON.stringify(history, null, 2));
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
private hash(data: string) {
|
|
152
|
+
return crypto.createHash("sha256").update(data).digest("hex");
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/* ------------------------------------------------------------ */
|
|
156
|
+
/* Diagnostics */
|
|
157
|
+
/* ------------------------------------------------------------ */
|
|
158
|
+
|
|
159
|
+
getHistory(): MigrationRecord[] {
|
|
160
|
+
return this.readHistory();
|
|
161
|
+
}
|
|
162
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { LioranDB } from "./database.js";
|
|
2
|
+
|
|
3
|
+
export type MigrationVersion = string;
|
|
4
|
+
|
|
5
|
+
export type MigrationFn = (db: LioranDB) => Promise<void>;
|
|
6
|
+
|
|
7
|
+
export interface Migration {
|
|
8
|
+
from: MigrationVersion;
|
|
9
|
+
to: MigrationVersion;
|
|
10
|
+
run: MigrationFn;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export interface MigrationRecord {
|
|
14
|
+
version: MigrationVersion;
|
|
15
|
+
appliedAt: number;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export interface MigrationMeta {
|
|
19
|
+
id?: string;
|
|
20
|
+
currentVersion: MigrationVersion;
|
|
21
|
+
history: MigrationRecord[];
|
|
22
|
+
}
|