datafolder 1.0.18 → 1.0.19

package/src/QyCache.js

@@ -1,78 +1,83 @@
 import {
-    join as f
+    join as v
 } from "node:path";
 
 import r from "node:events";
 
 import {
-    QyDir as D
+    QyDir as l
 } from "./QyDir.js";
 
 import {
-    QyFileLogger as c
+    QyFileLogger as g
 } from "./QyFileLogger.js";
 
 import {
-    QyKVData as F
+    QyKVData as h
 } from "./QyKVData.js";
 
 import {
-    QyTriggerNode as v,
+    QyTriggerNode as R,
     updateTriggerNodes as s
 } from "./QyTrigger.js";
 
 import {
     isString as C,
-    lockExclusiveFilePath as R,
-    unlockExclusiveFilePath as i
+    lockExclusiveFilePath as x,
+    unlockExclusiveFilePath as o,
+    ensureDir as q,
+    SpecialFilePaths as e,
+    SpecialOperators as t
 } from "./QyUtils.js";
 
 import {
-    dumpStructure as x
+    dumpStructure as O
 } from "./QyStructure.js";
 
 import {
     getIndexPropPaths as n,
-    parseNamePathAndQuery as g
+    parseNamePathAndQuery as d
 } from "./QyIndex.js";
 
 import {
-    getDefaultOptions as m
+    getDefaultOptions as u
 } from "./QyDefaultOptions.js";
 
 import {
-    ensureDir as q,
-    SpecialFilePaths as e
-} from "./QyUtils.js";
-
-import {
-    QueryCounter as l
+    QueryCounter as p
 } from "./QyUnionNode.js";
 
 import {
-    QyKVDataLoader as b
+    QyKVDataLoader as P
 } from "./QyKVDataLoader.js";
 
 import {
-    requireFromModCode as d
+    requireFromModCode as c
 } from "./QyMod.js";
 
 import {
-    logger as O
+    logger as b
 } from "./QyLogger.js";
 
+import {
+    QyEmbeddings as D
+} from "./QyEmbeddings.js";
+
+var {} = t;
+
 let {
     HiddenFolderName: y,
-    IndexDirPath: P,
-    TriggerDirPath: Q,
-    RpcDirPath: w,
-    ExpireDirPath: S,
-    MetaFilePath: t,
-    EmbeddingDirPath: L,
-    H2FDirPath: E
+    IndexDirPath: F,
+    TriggerDirPath: w,
+    RpcDirPath: E,
+    ExpireDirPath: m,
+    MetaFilePath: i,
+    HNSWDirPath: f,
+    InfoDirPath: Q,
+    RecycleDirPath: S
 } = e;
 
-class o extends r {
+class a extends r {
     rootFolder;
     options;
     qyDB;
@@ -83,42 +88,35 @@ class o extends r {
     rpcRootDir;
     indexRootDir;
     expireRootDir;
-    embeddingRootDir;
-    h2FRootDir;
+    hnswRootDir;
+    infoRootDir;
+    recycleRootDir;
     exclusiveFilePath;
     qyKVData;
     qyFileLogger;
+    qyEmbeddings;
     rpcFunMap = {};
     constructor(r, e, t) {
-        super(), e = {
-            ...m("QyCache"),
+        super(), this.rootFolder = r, e = this.options = {
+            ...u("QyCache"),
             ...e
-        }, o = new D("", !0, void 0, i = this), a = S.at(-1), s = P.at(-1), n = L.at(-1), 
-        g = E.at(-1), p = new D(y, !0, o), l = new D(a, !0, p), d = new D(s, !0, p), 
-        h = new D(n, !0, p), u = new D(g, !0, p), o._subdirMap = {
-            [y]: p
-        }, p._subdirMap = {
-            [a]: l,
-            [s]: d,
-            [n]: h,
-            [g]: u
-        }, Object.assign(i, {
-            rootDir: o,
-            expireRootDir: l,
-            indexRootDir: d,
-            embeddingRootDir: h,
-            h2FRootDir: u
-        });
-        var i, o, a, s, n, g, l, d, h, u, p = f(r, "qykv");
+        }, this.qyDB = t;
+        var i, o = this, t = o.rootDir = new l("", !0, void 0, o), a = new l(y, !0, t);
+        t._subdirMap = {
+            [y]: a
+        }, a._subdirMap = {};
+        for ([ s, i ] of [ [ F, "indexRootDir" ], [ m, "expireRootDir" ], [ f, "hnswRootDir" ], [ Q, "infoRootDir" ], [ S, "recycleRootDir" ] ]) {
+            var s = s.at(-1), n = new l(s, !0, a);
+            a._subdirMap[s] = n, o[i] = n;
+        }
+        t = v(r, "qykv");
         Object.assign(this, {
-            rootFolder: r,
-            options: e,
-            qyDB: t,
-            kvFolder: p,
-            rpcFolder: f(r, "rpc"),
-            exclusiveFilePath: f(r, ".opened"),
-            qyKVData: new F(p, e, this),
-            qyFileLogger: new c(f(r, "qylog"), e)
+            kvFolder: t,
+            rpcFolder: v(r, "rpc"),
+            exclusiveFilePath: v(r, ".opened"),
+            qyKVData: new h(t, e, this),
+            qyFileLogger: new g(v(r, "qylog"), e),
+            qyEmbeddings: new D(this, e)
         });
     }
     async start() {
@@ -129,20 +127,21 @@ class o extends r {
             qyKVData: i,
             qyFileLogger: o,
             exclusiveFilePath: a,
-            rootDir: s
+            rootDir: s,
+            qyEmbeddings: n
         } = this;
-        O.log(`Starting at ${r}...`);
+        b.log(`Starting at ${r}...`);
         try {
             await q(e), o.start();
             var [ {
-                maxChangeId: n,
+                maxChangeId: l,
                 loadedMaxAclNum: g,
-                qySnapshots: l,
+                qySnapshots: h,
                 aclKeyValueMap: d
-            } ] = await Promise.all([ ((r, e) => (r = new b(r, e)).load())(e, t), R(a) ]);
-            i.start(n, g, l, d);
+            } ] = await Promise.all([ ((r, e) => (r = new P(r, e)).load())(e, t), x(a) ]);
+            i.start(l, g, h, d), await n.start();
         } catch (r) {
-            throw o.stop(), r;
+            throw o.stop(), n.stop(), r;
         }
         t.preloadFoldersAtStart && !function r(e) {
             let {
@@ -151,48 +150,50 @@ class o extends r {
             } = e;
             for (var o of t) r(o);
         }(s);
-        var h = this, e = h.rootDir;
-        e.triggerNodes = [ new v() ];
-        for (u of (e = h.triggerRootDir = e.searchOrCreateSubdir(Q)).fileList) {
+        var u = this, e = u.rootDir;
+        e.triggerNodes = [ new R() ];
+        for (p of (e = u.triggerRootDir = e.searchOrCreateSubdir(w)).fileList) {
             var {
-                name: u,
-                fileContent: p
-            } = u, [ p, D, c ] = p;
-            h.insertTriggerToTree(u, p, D, c);
+                name: p,
+                fileContent: c
+            } = p, [ c, D, y ] = c;
+            u.insertTriggerToTree(p, c, D, y);
         }
         var F = this;
-        F.rpcRootDir = F.rootDir.searchOrCreateSubdir(w);
+        F.rpcRootDir = F.rootDir.searchOrCreateSubdir(E);
         for (m of F.rpcRootDir.fileList) {
             var {
                 name: m,
-                fileContent: y
+                fileContent: f
             } = m;
-            C(y) && F.insertRpc(m, y);
+            C(f) && F.insertRpc(m, f);
         }
-        return t.dumpStructureAtStart && await x(f(r, "qystruct"), s), O.log("Started."), 
+        return t.dumpStructureAtStart && await O(v(r, "qystruct"), s), b.log("Started."), 
         this;
     }
     async stop() {
-        O.log("Stopping...");
+        b.log("Stopping...");
         var {
             qyKVData: r,
-            qyFileLogger: e,
-            exclusiveFilePath: t
+            qyEmbeddings: e,
+            qyFileLogger: t,
+            exclusiveFilePath: i
         } = this;
-        return await r.stop(), O.log("QyFileLogger stopped."), await e.stop(), i(t), 
-        Object.assign(this, {
+        return await r.stop(), await e.stop(), b.log("QyFileLogger stopped."), await t.stop(), 
+        o(i), Object.assign(this, {
             rootDir: void 0,
             indexRootDir: void 0,
             expireRootDir: void 0,
             triggerRootDir: void 0,
             rpcRootDir: void 0,
-            embeddingRootDir: void 0,
-            h2FRootDir: void 0,
+            hnswRootDir: void 0,
+            infoRootDir: void 0,
+            recycleRootDir: void 0,
             qyKVData: void 0,
             rpcFunMap: void 0,
             qyDB: void 0,
             qyFileLogger: void 0
-        }), O.log("Stopped."), this;
+        }), b.log("Stopped."), this;
     }
     increaseChangeId() {
         return this.qyKVData.increaseChangeId();
@@ -210,20 +211,20 @@ class o extends r {
     getFile(r, e = this.rootDir) {
         return r.isFile ? r : C(r) ? e.getFile(r) : e.searchFile(r);
     }
-    getDir(r, e = this.rootDir) {
-        return r.isDir ? r : C(r) ? e.getSubdir(r) : e.searchSubdir(r);
-    }
     getOrCreateFile(r, e = this.rootDir) {
         return r.isFile ? r : C(r) ? e.getOrCreateFile(r) : e.searchOrCreateFile(r);
     }
+    getDir(r, e = this.rootDir) {
+        return r.isDir ? r : C(r) ? e.getSubdir(r) : e.searchSubdir(r);
+    }
     getOrCreateDir(r, e = this.rootDir) {
         return r.isDir ? r : C(r) ? e.getOrCreateSubdir(r) : e.searchOrCreateSubdir(r);
     }
     getIndexDir(r) {
-        return this.indexRootDir.searchSubdir(p(r.fullPathHash));
+        return this.indexRootDir.searchSubdir(K(r.fullPathHash));
     }
     getOrCreateIndexDir(r) {
-        return this.indexRootDir.searchOrCreateSubdir(p(r.fullPathHash));
+        return this.indexRootDir.searchOrCreateSubdir(K(r.fullPathHash));
     }
     getTriggerFile(r) {
         return this.triggerRootDir.getFile(r);
@@ -232,29 +233,36 @@ class o extends r {
         return this.triggerRootDir.getOrCreateFile(r);
     }
     getExpireInfoFile(r, e = "c") {
-        return this.expireRootDir.searchFile(u(r, e));
+        return this.expireRootDir.searchFile(L(r, e));
     }
     getOrCreateExpireInfoFile(r, e = "c") {
-        return this.expireRootDir.searchOrCreateFile(u(r, e));
+        return this.expireRootDir.searchOrCreateFile(L(r, e));
     }
-    getH2FFile(r) {
-        return this.h2FRootDir.searchFile(a(r));
+    getInfoFile(r) {
+        return this.infoRootDir.searchFile(I(r.fullPathHash));
     }
-    getOrCreateH2FFile(r) {
-        return this.h2FRootDir.searchOrCreateFile(a(r));
+    getOrCreateInfoFile(r) {
+        return this.infoRootDir.searchOrCreateFile(I(r.fullPathHash));
     }
-    h2F(r) {
-        r = this.getH2FFile(r);
-        if (r) return this.getOrCreateFile(r.fileContent);
+    hToSrcFile(r) {
+        r = this.infoRootDir.searchFile(I(r));
+        if (r && r.fileContent) {
+            r = r.fileContent.namePath;
+            if (r) return this.getOrCreateFile(r);
+        }
     }
-    getEmbeddingFile(r) {
-        return this.embeddingRootDir.searchFile(h(r));
+    getEmbedding(r) {
+        var e, t = this.getInfoFile(r);
+        if (t && t.fileContent) return {
+            length: t,
+            idx: e
+        } = t.fileContent, this.qyEmbeddings.getEmbedding(r.fullPathHash, t, e);
     }
-    getOrCreateEmbeddingFile(r) {
-        return this.embeddingRootDir.searchOrCreateFile(h(r));
+    getOrCreateRecycleFile(r) {
+        return this.recycleRootDir.getOrCreateFile(r);
     }
-    getEmbedding(r) {
-        return this.embeddingRootDir.searchFile(h(r))?.fileContent;
+    getRecycleFile(r) {
+        return this.recycleRootDir.getFile(r);
     }
     get triggerFileList() {
         return this.triggerRootDir.fileList;
@@ -269,13 +277,13 @@ class o extends r {
         return this.rpcRootDir.fileList;
     }
     get metaFile() {
-        return this.rootDir.searchOrCreateFile(t);
+        return this.rootDir.searchOrCreateFile(i);
     }
     insertTriggerToTree(r, e, t, i) {
         var {
             rootDir: o,
             qyDB: a
-        } = this, i = d("trigger", r, i, a);
+        } = this, i = c("trigger", r, i, a);
         o.triggerNodes[0].insertTrigger(r, e, t, i), s(o, e);
     }
     removeTriggerFromTree(r, e) {
@@ -286,28 +294,28 @@ class o extends r {
         var {
             rpcFunMap: t,
             qyDB: i
-        } = this, e = d("rpc", r, e, i);
+        } = this, e = c("rpc", r, e, i);
         e && (t[r] = e);
     }
     removeRpc(r) {
         delete this.rpcFunMap[r];
     }
     queryFiles(r, e, t, i, o = this.rootDir) {
-        var a = [], t = (C(e) && !t && (t = 1), new l(t, i)), i = this.options.queryAutoCreateIndex;
-        return I(r, e, a, t, o, i), a;
+        var a = [], t = (C(e) && !t && (t = 1), new p(t, i)), i = this.options.queryAutoCreateIndex;
+        return T(r, e, a, t, o, i), a;
     }
     queryFilesMulti(r, e, t, i = this.rootDir) {
-        var o, a, s = [], n = new l(e, t), g = this.options.queryAutoCreateIndex;
-        for ([ o, a ] of r) I(o, a, s, n, i, g);
+        var o, a, s = [], n = new p(e, t), l = this.options.queryAutoCreateIndex;
+        for ([ o, a ] of r) T(o, a, s, n, i, l);
         return s;
     }
     async callRpc(e, r, t) {
         var i, o = this.rpcFunMap[e];
-        if (!o) return O.warn(i = `RPC ${e} not implemented.`), new Error(i);
+        if (!o) return b.warn(i = `RPC ${e} not implemented.`), new Error(i);
         try {
             return await o(r, t);
         } catch (r) {
-            return O.error(`Calling RPC ${e} failed:`, r), r;
+            return b.error(`Calling RPC ${e} failed:`, r), r;
         }
     }
     monitorItemExpire(r, e) {
@@ -324,26 +332,22 @@ class o extends r {
     }
 }
 
-function a(r) {
-    return [ r.substring(0, 2), r ];
-}
-
-function h(r) {
+function I(r) {
     return [ r.substring(0, 2), r ];
 }
 
-function u(r, e) {
+function L(r, e) {
     return [ e, r.substring(0, 2), r ];
 }
 
-function p(r) {
+function K(r) {
     return [ r[0], r ];
 }
 
-function I(r, e, t, i, o, a) {
-    e && (a = a ? n(e) : void 0, g(r, o, a, e, t, i));
+function T(r, e, t, i, o, a) {
+    e && (a = a ? n(e) : void 0, d(r, o, a, e, t, i));
 }
 
 export {
-    o as QyCache
+    a as QyCache
 };

package/src/QyDB.js

@@ -47,10 +47,7 @@ class l extends t {
             ...o("QyDB"),
             ...t
         }, this.rootFolder = e;
-        for (let i of [ "createIndex", "delIndex", "insert", "put", "remove", "delDir", "delFile", "delContent", "insertTrigger", "removeTrigger", "insertRpc", "removeRpc", "insertEmbedding", "delEmbedding", "expireFile", "expireDir", "expireContent", "expireFileM", "expireDirM", "expireContentM", "removeFileExpire", "removeDirExpire", "removeContentExpire" ]) this[i] = (...e) => {
-            var t = this.currentBatch;
-            return t[i].apply(t, e).run();
-        };
+        for (let t of [ "createIndex", "delIndex", "insert", "put", "remove", "delDir", "delFile", "delContent", "insertTrigger", "removeTrigger", "insertRpc", "removeRpc", "insertEmbedding", "delEmbedding", "insertWithEmbedding", "expireFile", "expireDir", "expireContent", "expireFileM", "expireDirM", "expireContentM", "removeFileExpire", "removeDirExpire", "removeContentExpire" ]) this[t] = (...e) => this.currentBatch[t](...e).run();
     }
     async start(e) {
         var t;

package/src/QyDefaultOptions.js

@@ -1,5 +1,6 @@
 var e = {
     append: !1,
+    read: !1,
     retryIntervalFactor: 2,
     maxRetryInterval: 60,
     emitSaveEvent: !1,
@@ -57,6 +58,13 @@ let t = {
     },
     QyAclWriter: {
         minBatchSize: 2
+    },
+    QyEmbeddingWriter: {
+        append: !0,
+        read: !0
+    },
+    QyEmbeddings: {
+        appendOnly: !1
     }
 };
 

package/src/QyEmbeddingWriter.d.ts

@@ -0,0 +1,12 @@
+import { QyBinWriter } from "./QyBinWriter.ts";
+export declare class QyEmbeddingWriter extends QyBinWriter {
+    length: any;
+    byteLength: any;
+    totalCount: number;
+    constructor(filePath: any | undefined, length: any);
+    start(): Promise<void>;
+    append(embedding: Float32Array, needPromise?: any): any;
+    pWrite(embedding: Float32Array, idx: any, needPromise?: any): any;
+    _op_pWrite(buffer: any, idx: any): any;
+    readSync(idx: any): Float32Array<ArrayBuffer> | undefined;
+}

package/src/QyEmbeddingWriter.js

@@ -0,0 +1,55 @@
+import {
+    readBufferFromFD as s
+} from "./QyUtils.js";
+
+import {
+    getDefaultOptions as r
+} from "./QyDefaultOptions.js";
+
+import {
+    QyBinWriter as t
+} from "./QyBinWriter.js";
+
+class e extends t {
+    length;
+    byteLength;
+    totalCount = 0;
+    constructor(t = void 0, e) {
+        super(r("QyEmbeddingWriter"), t), this.length = e, this.byteLength = e * Float32Array.BYTES_PER_ELEMENT;
+    }
+    async start() {
+        await this.pushAndRunWithPromise({
+            opFunName: "_op_start"
+        }), 0 == this.totalCount && (this.totalCount = this.totalSavedSize / this.byteLength);
+    }
+    append(t, e = !1) {
+        return ++this.totalCount, this.save(0, i(t), e);
+    }
+    pWrite(t, e, r = !1) {
+        t = {
+            opFunName: "_op_pWrite",
+            args: [ i(t), e ]
+        };
+        return r ? this.pushAndRunWithPromise(t) : this.pushAndRun(t);
+    }
+    _op_pWrite(t, e) {
+        return this.fileHandle.write(t, 0, t.byteLength, e * this.byteLength);
+    }
+    readSync(t) {
+        var {
+            filePath: e,
+            fileHandle: r,
+            length: i,
+            byteLength: n
+        } = this;
+        return (r = r && s(r.fd, e, t * n, n)) ? new Float32Array(r.buffer, r.byteOffset, i) : void 0;
+    }
+}
+
+function i(t) {
+    return Buffer.from(t.buffer, t.byteOffset, t.byteLength);
+}
+
+export {
+    e as QyEmbeddingWriter
+};

package/src/QyEmbeddings.d.ts

@@ -0,0 +1,15 @@
+export declare class QyEmbeddings {
+    qyCache: any;
+    embeddingFolder: any;
+    options: any;
+    qyDB: any;
+    qyEmbeddingWriters: any;
+    hashToEmbeddingMap: {};
+    constructor(qyCache: any, options?: any);
+    start(): Promise<void>;
+    stop(): Promise<void>;
+    addEmbedding(fullPathHash: any, embedding: any): any;
+    getEmbedding(fullPathHash: any, length: any, idx: any): any;
+    updateEmbedding(fullPathHash: any, embedding: any, idx: any): any;
+    delEmbedding(fullPathHash: any, length: any, idx: any): void;
+}

package/src/QyEmbeddings.js

@@ -0,0 +1,128 @@
+import {
+    readdir as t
+} from "node:fs/promises";
+
+import {
+    folderOrFileExists as i,
+    SpecialOperators as e
+} from "./QyUtils.js";
+
+import {
+    getDefaultOptions as r
+} from "./QyDefaultOptions.js";
+
+import {
+    logger as a
+} from "./QyLogger.js";
+
+import {
+    basename as n,
+    join as s
+} from "node:path";
+
+import {
+    QyEmbeddingWriter as d
+} from "./QyEmbeddingWriter.js";
+
+let o = e.$del;
+
+class l {
+    qyCache;
+    embeddingFolder;
+    options;
+    qyDB;
+    qyEmbeddingWriters = {};
+    hashToEmbeddingMap = {};
+    constructor(e, t = void 0) {
+        var i = s(e.rootFolder, "qyembedding");
+        t = {
+            ...r("QyEmbeddings"),
+            ...t
+        }, Object.assign(this, {
+            qyCache: e,
+            embeddingFolder: i,
+            options: t,
+            qyDB: e.qyDB
+        });
+    }
+    async start() {
+        var e = this.embeddingFolder;
+        await i(e) && await Promise.all((await t(e)).filter(e => /[0-9]+\.bin$/.test(e)).map(async e => {
+            await h(this, n(e, ".bin")).start();
+        }));
+    }
+    async stop() {
+        var e = this.qyEmbeddingWriters;
+        e && (await Promise.all(Object.values(e).map(e => e.stop())), this.qyEmbeddingWriters = void 0);
+    }
+    addEmbedding(e, t) {
+        var {
+            hashToEmbeddingMap: i,
+            options: r
+        } = this;
+        if (i[e] = t, !r.appendOnly) {
+            i = ((e, t) => {
+                var {
+                    qyCache: e,
+                    qyDB: i
+                } = e, r = e.getRecycleFile(t);
+                if (r) {
+                    var n, e = r.fileContent;
+                    for (n in e) return i.insert(r, {
+                        [n]: o
+                    }), n;
+                }
+            })(this, t.length);
+            if (null != i) return p(this, e, t, i);
+        }
+        return m(this, e, t);
+    }
+    getEmbedding(e, t, i) {
+        var r = this.hashToEmbeddingMap, r = r[e];
+        return r || (null != t && null != i ? this.hashToEmbeddingMap[e] = g(this, t).readSync(i) : void 0);
+    }
+    updateEmbedding(e, t, i) {
+        var {
+            options: r,
+            hashToEmbeddingMap: n
+        } = this;
+        return n[e] = t, r.appendOnly ? m(this, e, t) : p(this, e, t, i);
+    }
+    delEmbedding(e, t, i) {
+        var {
+            hashToEmbeddingMap: r,
+            qyCache: n,
+            qyDB: a
+        } = this;
+        delete r[t], null != t && null != i && a.insert(n.getOrCreateRecycleFile(t), {
+            [i]: 1
+        });
+    }
+}
+
+function m(e, t, i) {
+    var r = i.length, e = g(e, r), n = e.totalCount;
+    return e.append(i), a.info(`Append ${t} ${r} ` + n), n;
+}
+
+function p(e, t, i, r) {
+    var n = i.length;
+    return g(e, i.length).pWrite(i, r), a.info(`pWrite ${t} ${n} ` + r), r;
+}
+
+function h(e, t) {
+    var {
+        qyEmbeddingWriters: e,
+        embeddingFolder: i
+    } = e;
+    return e[t] = new d(s(i, t + ".bin"), t);
+}
+
+function g(e, t) {
+    let i = e.qyEmbeddingWriters[t];
+    return i || (i = h(e, t)).start(), i;
+}
+
+export {
+    l as QyEmbeddings
+};