datafolder 1.0.17 → 1.0.19

package/src/QyBinWriter.d.ts

@@ -1,18 +1,14 @@
 import { QyOpRunner } from "./QyOpRunner.ts";
 export declare class QyBinWriter extends QyOpRunner {
-    _filePath: any;
-    _totalSavedSize: number;
+    filePath: any;
+    totalSavedSize: number;
     fileHandle: any;
-    constructor(options?: any, filePath?: any);
-    get filePath(): any;
-    set filePath(val: any);
-    get totalSavedSize(): number;
-    set totalSavedSize(val: number);
-    save(changeId: any, data: any, needPromise: any): any;
+    constructor(options?: any, filePath?: any, callOpNameList?: any, castOpNameList?: any);
+    save(changeId: any, data?: any, needPromise?: any): any;
     truncate(): any;
     switch(newFilePath: any): any;
     getBinData(dataQueue: any[], changeId?: undefined): any;
-    _op_start(filePath: any): Promise<this>;
+    _op_start(filePath?: any): Promise<this>;
     _op_stop(): Promise<void>;
     _op_save(changeId: any, dataQueue: any): Promise<void> | undefined;
     _op_truncate(): Promise<void>;

package/src/QyBinWriter.js

@@ -1,5 +1,5 @@
 import {
-    open as s
+    open as n
 } from "node:fs/promises";
 
 import {
@@ -7,16 +7,18 @@ import {
 } from "./QyOpRunner.js";
 
 import {
-    ensureParentDir as n,
-    getSize as o,
+    ensureParentDir as l,
+    getSize as f,
     getSizeStr as v,
     sleep as _,
-    setPromise as l,
-    toBuffer as S
+    setPromise as s,
+    toBuffer as S,
+    folderOrFileExists as p,
+    isString as r
 } from "./QyUtils.js";
 
 import {
-    getDefaultOptions as a
+    getDefaultOptions as o
 } from "./QyDefaultOptions.js";
 
 import {
@@ -24,32 +26,20 @@ import {
 } from "./QyLogger.js";
 
 class e extends t {
-    _filePath;
-    _totalSavedSize = 0;
+    filePath;
+    totalSavedSize = 0;
     fileHandle;
-    constructor(t = void 0, e = void 0) {
-        super({
-            ...a("QyBinWriter"),
+    constructor(t = void 0, e = void 0, a = void 0, i = void 0) {
+        r(t) && ([ e, t ] = [ t, void 0 ]), super({
+            ...o("QyBinWriter"),
             ...t
-        }), this.filePath = e, f(this);
+        }, a, i), this.filePath = e, h(this);
     }
-    get filePath() {
-        return this._filePath;
-    }
-    set filePath(t) {
-        this._filePath = t;
-    }
-    get totalSavedSize() {
-        return this._totalSavedSize;
-    }
-    set totalSavedSize(t) {
-        this._totalSavedSize = t;
-    }
-    save(t, e, a) {
+    save(t, e = void 0, a = !1) {
         null == e && ([ e, t ] = [ t, 0 ]);
         var i, r = this.lastOp;
         return "_op_save" === r?.opFunName ? (i = r.args, i[0] = t, i[1].push(e), 
-        a && l(r), r.promise) : (i = {
+        a && s(r), r.promise) : (i = {
             opFunName: "_op_save",
             args: [ t, [ e ] ]
         }, a ? this.pushAndRunWithPromise(i) : this.pushAndRun(i));
@@ -71,7 +61,7 @@ class e extends t {
         var a = (t = t.flat()).length;
         return 0 == a ? "" : 1 == a ? t[0] : Buffer.isBuffer(t[0]) ? Buffer.concat(t) : Buffer.concat(t.map(t => Buffer.from(t)));
     }
-    async _op_start(t) {
+    async _op_start(t = void 0) {
         return await i(this, t = t || this.filePath), this;
     }
     async _op_stop() {
@@ -84,7 +74,7 @@ class e extends t {
         } catch (t) {
             g.error(`Close ${e} failed:`, t);
         }
-        f(this);
+        h(this);
     }
     _op_save(t, e) {
         e = this.getBinData(e, t);
@@ -93,24 +83,22 @@ class e extends t {
                 fileHandle: i,
                 totalSavedSize: r,
                 filePath: s,
-                options: n
+                options: o
             } = t, {
-                emitSaveEvent: o,
+                emitSaveEvent: n,
                 maxFileSize: l,
                 maxRetryInterval: f,
-                retryIntervalFactor: h
-            } = n, p = S(a), u = p.length, c = `size ${v(u)} (MaxId:${e}) to ` + s, d;
+                retryIntervalFactor: p
+            } = o, h = S(a), u = h.length, d = `size ${v(u)} (MaxId:${e}) to ` + s, c;
             for (;"_op_truncate" !== t.firstOp?.opFunName; ) try {
-                await i.appendFile(p, {
-                    flush: !0
-                });
+                await i.write(h, 0, h.byteLength, r);
                 var m = t.totalSavedSize = r + u;
-                return o && 0 < e && t.emit("save", e), void (!t.maxFileSizeEmitted && l && m > l && (t.maxFileSizeEmitted = !0, 
+                return n && 0 < e && t.emit("save", e), void (!t.maxFileSizeEmitted && l && m > l && (t.maxFileSizeEmitted = !0, 
                 t.emit("maxFileSize", m)));
             } catch (t) {
-                g.error(`Saving ${c} failed:`, t, "Truncated back to " + r), await i.truncate(r), 
-                d = d ? Math.min(f, d * h) : 1, g.log(`Retry in ${d} seconds`), 
-                await _(d);
+                g.error(`Saving ${d} failed:`, t, "Truncated back to " + r), await i.truncate(r), 
+                c = c ? Math.min(f, c * p) : 1, g.log(`Retry in ${c} seconds`), 
+                await _(c);
             }
         })(this, t, e);
     }
@@ -124,7 +112,7 @@ class e extends t {
         } catch (t) {
             g.error(`Truncate ${e} failed:`, t);
         }
-        f(this);
+        h(this);
     }
     async _op_switch(t) {
         var {
@@ -140,17 +128,20 @@ async function i(t, e) {
         fileHandle: a,
         options: i
     } = t;
-    f(t);
+    h(t);
     try {
-        a && (a.close(), t.fileHandle = void 0), await n(e);
-        var r = i.append;
-        t.fileHandle = await s(e, r ? "a" : "w"), t.filePath = e, r && (t.totalSavedSize = await o(e));
+        a && (a.close(), t.fileHandle = void 0), await l(e), t.filePath = e;
+        var {
+            append: r,
+            read: s
+        } = i, o = r && await p(e) ? "r+" : s ? "w+" : "w";
+        t.fileHandle = await n(e, o), r && (t.totalSavedSize = await f(e));
     } catch (t) {
         g.error(`Open ${e} failed:`, t);
     }
 }
 
-function f(t) {
+function h(t) {
     Object.assign(t, {
         totalSavedSize: 0,
         maxFileSizeEmitted: !1

package/src/QyBinaryHeap.d.ts

@@ -0,0 +1,10 @@
+export declare class QyBinaryHeap {
+    items: any[];
+    cmpFun: any;
+    constructor(cmpFun?: (x: any, y: any) => number);
+    get size(): number;
+    get isEmpty(): boolean;
+    peek(): any;
+    push(item: any): void;
+    pop(): any;
+}

package/src/QyBinaryHeap.js

@@ -0,0 +1,55 @@
+class t {
+    items = [];
+    cmpFun;
+    constructor(t = (t, e) => t - e) {
+        this.cmpFun = t;
+    }
+    get size() {
+        return this.items.length;
+    }
+    get isEmpty() {
+        return 0 == this.items.length;
+    }
+    peek() {
+        return this.items[0];
+    }
+    push(t) {
+        var e = this.items;
+        e.push(t), i(this);
+    }
+    pop() {
+        if (!this.isEmpty) {
+            for (var t = this.items, e = t[0], [ t, i = 0 ] = (t[0] = t.at(-1), 
+            --t.length, [ this ]), {
+                items: s,
+                cmpFun: r
+            } = t, h = s.length; i < h; ) {
+                var n = 2 * i + 1;
+                if (!(n < h)) break;
+                {
+                    let t = i;
+                    0 < r(s[n], s[t]) && (t = n);
+                    n = 1 + n;
+                    if ((t = n < h && 0 < r(s[n], s[t]) ? n : t) == i) break;
+                    [ s[i], s[t] ] = [ s[t], s[i] ], i = t;
+                }
+            }
+            return e;
+        }
+    }
+}
+
+function i(t, e = t.items.length - 1) {
+    for (var {
+        items: i,
+        cmpFun: s
+    } = t; 0 < e; ) {
+        var r = Math.floor((e - 1) / 2);
+        if (!(0 < s(i[e], i[r]))) break;
+        [ i[e], i[r] ] = [ i[r], i[e] ], e = r;
+    }
+}
+
+export {
+    t as QyBinaryHeap
+};

package/src/QyCD.d.ts

@@ -13,6 +13,9 @@ export declare class QyCD {
     indexDirP(indexHostDirNamePath: string | string[] | QyDir, namePath: string | string[], parentDir?: any): QyDir | undefined;
     file(namePath: string | string[], parentDir?: any): QyFile | undefined;
     fileP(namePath: string | string[], parentDir?: any): QyFile | undefined;
+    fileContent(namePath: string | string[], parentDir?: any): any;
+    infoFile(namePath: string | string[], parentDir?: any): any;
+    embedding(namePath: string | string[], parentDir?: any): any;
     triggerFile(triggerName: any): any;
     get triggerFileList(): any;
     rpcFile(rpcName: any): any;

package/src/QyCD.js

@@ -1,6 +1,6 @@
 import {
-    isString as t,
-    pathSplit as s
+    isString as i,
+    pathSplit as n
 } from "./QyUtils.js";
 
 class r {
@@ -10,32 +10,45 @@ class r {
         this.qyCache = r;
     }
     cd(r, e = this.currentDir) {
-        return t(r) && (r = s(r)), this.cdP(r, e);
+        return i(r) && (r = n(r)), this.cdP(r, e);
     }
     cdP(r, e = this.currentDir) {
-        return this.currentDir = this.qyCache?.getOrCreateDir(r, e), this;
+        return this.currentDir = this.qyCache.getOrCreateDir(r, e), this;
     }
     dir(r, e = this.currentDir) {
-        return t(r) && (r = s(r)), this.dirP(r, e);
+        return i(r) && (r = n(r)), this.dirP(r, e);
     }
     dirP(r, e = this.currentDir) {
-        return this.qyCache?.getDir(r, e);
+        return this.qyCache.getDir(r, e);
     }
-    indexDir(r, e, i = this.currentDir) {
-        return t(r) && (r = s(r)), t(e) && (e = s(e)), this.indexDirP(r, e, i);
+    indexDir(r, e, t = this.currentDir) {
+        return i(r) && (r = n(r)), i(e) && (e = n(e)), this.indexDirP(r, e, t);
     }
-    indexDirP(r, e, i = this.currentDir) {
-        var t = this.qyCache, r = t?.getDir(r, i);
+    indexDirP(r, e, t = this.currentDir) {
+        var i = this.qyCache, r = i.getDir(r, t);
         if (r) {
-            i = t?.getIndexDir(r);
-            if (i) return e && 0 != e.length ? t?.getDir(e, i) : i;
+            t = i.getIndexDir(r);
+            if (t) return e && 0 != e.length ? i.getDir(e, t) : t;
         }
     }
     file(r, e = this.currentDir) {
-        return t(r) && (r = s(r)), this.fileP(r, e);
+        return i(r) && (r = n(r)), this.fileP(r, e);
     }
     fileP(r, e = this.currentDir) {
-        return this.qyCache?.getFile(r, e);
+        return this.qyCache.getFile(r, e);
+    }
+    fileContent(r, e = this.currentDir) {
+        return this.file(r, e)?.fileContent;
+    }
+    infoFile(r, e = this.currentDir) {
+        i(r) && (r = n(r));
+        var t = this.qyCache;
+        return t.getInfoFile(t.getOrCreateFile(r, e));
+    }
+    embedding(r, e = this.currentDir) {
+        i(r) && (r = n(r));
+        var t = this.qyCache, r = t.getOrCreateFile(r, e);
+        return t.getEmbedding(r);
     }
     triggerFile(r) {
         return this.qyCache.getTriggerFile(r);
@@ -49,55 +62,55 @@ class r {
     get rpcFileList() {
         return this.qyCache.rpcFileList;
     }
-    view(r, e = void 0, i = this.currentDir) {
-        return t(r) && (r = s(r)), this.viewP(r, e, i);
+    view(r, e = void 0, t = this.currentDir) {
+        return i(r) && (r = n(r)), this.viewP(r, e, t);
     }
-    viewP(r, e, i = this.currentDir) {
-        return this.qyCache.getFile(r, i)?.view(e);
+    viewP(r, e, t = this.currentDir) {
+        return this.qyCache.getFile(r, t)?.view(e);
     }
-    queryFiles(r, e, i = void 0, t = void 0, s = this.currentDir) {
-        return this.qyCache.queryFiles(r, e, i, t, s);
+    queryFiles(r, e, t = void 0, i = void 0, n = this.currentDir) {
+        return this.qyCache.queryFiles(r, e, t, i, n);
     }
-    queryFilesMulti(r, e, i, t = this.currentDir) {
-        return this.qyCache.queryFilesMulti(r, e, i, t);
+    queryFilesMulti(r, e, t, i = this.currentDir) {
+        return this.qyCache.queryFilesMulti(r, e, t, i);
     }
-    queryTree(r, e, i, t, s, h = this.currentDir) {
-        return n(this.qyCache.queryFiles(r, e, t, s, h), i);
+    queryTree(r, e, t, i, n, s = this.currentDir) {
+        return h(this.qyCache.queryFiles(r, e, i, n, s), t);
     }
-    queryTreeMulti(r, e, i, t, s = this.currentDir) {
-        return n(this.qyCache.queryFilesMulti(r, i, t, s), e);
+    queryTreeMulti(r, e, t, i, n = this.currentDir) {
+        return h(this.qyCache.queryFilesMulti(r, t, i, n), e);
     }
     resetCurrentDir() {
         this.currentDir = void 0;
     }
     fileListToTree(r, e) {
-        return n(r, e);
+        return h(r, e);
     }
 }
 
-function n(r, i) {
-    var t, s = {};
-    for (t of r) {
-        var h, n = t.parentList;
-        let e = s;
-        for (h of n) {
+function h(r, t) {
+    var i, n = {};
+    for (i of r) {
+        var s, h = i.parentList;
+        let e = n;
+        for (s of h) {
             var {
                 name: u,
                 fullPathHash: c
-            } = h;
+            } = s;
             let r = e.subdirMap;
             r = r || (e.subdirMap = {}), e = (e = r[u]) || (r[u] = {
                 fullPathHash: c
             });
         }
         let r = e.fileMap;
-        (r = r || (e.fileMap = {}))[t.name] = {
-            fileContent: t.view(i, !0),
-            fileContentKey: t.fileContentKey
+        (r = r || (e.fileMap = {}))[i.name] = {
+            fileContent: i.view(t, !0),
+            fileContentKey: i.fileContentKey
         };
     }
     return {
-        tree: s,
+        tree: n,
         count: r.length
     };
 }

package/src/QyCache.d.ts

@@ -12,9 +12,13 @@ export declare class QyCache extends EventEmitter {
     rpcRootDir: any;
     indexRootDir: any;
     expireRootDir: any;
+    hnswRootDir: any;
+    infoRootDir: any;
+    recycleRootDir: any;
     exclusiveFilePath: any;
     qyKVData: any;
     qyFileLogger: any;
+    qyEmbeddings: any;
     rpcFunMap: {};
     constructor(rootFolder: string, options?: any, qyDB?: any);
     start(): Promise<this>;
@@ -23,8 +27,8 @@ export declare class QyCache extends EventEmitter {
     decreaseChangeId(): void;
     save(changeId: any, cmdArgAsListObj: any, logText: any, syncList: any, needPromise: any): any;
     getFile(namePath: QyFile | string | string[], parentDir?: any): QyFile | undefined;
-    getDir(namePath: QyDir | string | string[], parentDir?: any): QyDir | undefined;
     getOrCreateFile(namePath: QyFile | string | string[], parentDir?: any): QyFile;
+    getDir(namePath: QyDir | string | string[], parentDir?: any): QyDir | undefined;
     getOrCreateDir(namePath: QyDir | string | string[], parentDir?: any): QyDir;
     getIndexDir(indexHostDir: QyDir): any;
     getOrCreateIndexDir(indexHostDir: any): any;
@@ -32,6 +36,12 @@ export declare class QyCache extends EventEmitter {
     getOrCreateTriggerFile(triggerName: any): any;
     getExpireInfoFile(fullPathHash: any, type?: string): any;
     getOrCreateExpireInfoFile(fullPathHash: any, type?: string): any;
+    getInfoFile(srcFile: any): any;
+    getOrCreateInfoFile(srcFile: any): any;
+    hToSrcFile(fullPathHash: any): QyFile | undefined;
+    getEmbedding(srcFile: any): any;
+    getOrCreateRecycleFile(length: any): any;
+    getRecycleFile(length: any): any;
     get triggerFileList(): any;
     getRpcFile(rpcFunName: any): any;
     getOrCreateRpcFile(rpcFunName: any): any;