datafolder 1.0.18 → 1.0.19

package/src/QyBinWriter.d.ts

@@ -1,18 +1,14 @@
 import { QyOpRunner } from "./QyOpRunner.ts";
 export declare class QyBinWriter extends QyOpRunner {
-    _filePath: any;
-    _totalSavedSize: number;
+    filePath: any;
+    totalSavedSize: number;
     fileHandle: any;
-    constructor(options?: any, filePath?: any);
-    get filePath(): any;
-    set filePath(val: any);
-    get totalSavedSize(): number;
-    set totalSavedSize(val: number);
-    save(changeId: any, data: any, needPromise: any): any;
+    constructor(options?: any, filePath?: any, callOpNameList?: any, castOpNameList?: any);
+    save(changeId: any, data?: any, needPromise?: any): any;
     truncate(): any;
     switch(newFilePath: any): any;
     getBinData(dataQueue: any[], changeId?: undefined): any;
-    _op_start(filePath: any): Promise<this>;
+    _op_start(filePath?: any): Promise<this>;
     _op_stop(): Promise<void>;
     _op_save(changeId: any, dataQueue: any): Promise<void> | undefined;
     _op_truncate(): Promise<void>;

package/src/QyBinWriter.js

@@ -1,5 +1,5 @@
 import {
-    open as s
+    open as n
 } from "node:fs/promises";
 
 import {
@@ -7,16 +7,18 @@ import {
 } from "./QyOpRunner.js";
 
 import {
-    ensureParentDir as n,
-    getSize as o,
+    ensureParentDir as l,
+    getSize as f,
     getSizeStr as v,
     sleep as _,
-    setPromise as l,
-    toBuffer as S
+    setPromise as s,
+    toBuffer as S,
+    folderOrFileExists as p,
+    isString as r
 } from "./QyUtils.js";
 
 import {
-    getDefaultOptions as a
+    getDefaultOptions as o
 } from "./QyDefaultOptions.js";
 
 import {
@@ -24,32 +26,20 @@ import {
 } from "./QyLogger.js";
 
 class e extends t {
-    _filePath;
-    _totalSavedSize = 0;
+    filePath;
+    totalSavedSize = 0;
     fileHandle;
-    constructor(t = void 0, e = void 0) {
-        super({
-            ...a("QyBinWriter"),
+    constructor(t = void 0, e = void 0, a = void 0, i = void 0) {
+        r(t) && ([ e, t ] = [ t, void 0 ]), super({
+            ...o("QyBinWriter"),
             ...t
-        }), this.filePath = e, f(this);
+        }, a, i), this.filePath = e, h(this);
     }
-    get filePath() {
-        return this._filePath;
-    }
-    set filePath(t) {
-        this._filePath = t;
-    }
-    get totalSavedSize() {
-        return this._totalSavedSize;
-    }
-    set totalSavedSize(t) {
-        this._totalSavedSize = t;
-    }
-    save(t, e, a) {
+    save(t, e = void 0, a = !1) {
         null == e && ([ e, t ] = [ t, 0 ]);
         var i, r = this.lastOp;
         return "_op_save" === r?.opFunName ? (i = r.args, i[0] = t, i[1].push(e), 
-        a && l(r), r.promise) : (i = {
+        a && s(r), r.promise) : (i = {
             opFunName: "_op_save",
             args: [ t, [ e ] ]
         }, a ? this.pushAndRunWithPromise(i) : this.pushAndRun(i));
@@ -71,7 +61,7 @@ class e extends t {
         var a = (t = t.flat()).length;
         return 0 == a ? "" : 1 == a ? t[0] : Buffer.isBuffer(t[0]) ? Buffer.concat(t) : Buffer.concat(t.map(t => Buffer.from(t)));
     }
-    async _op_start(t) {
+    async _op_start(t = void 0) {
         return await i(this, t = t || this.filePath), this;
     }
     async _op_stop() {
@@ -84,7 +74,7 @@ class e extends t {
         } catch (t) {
             g.error(`Close ${e} failed:`, t);
         }
-        f(this);
+        h(this);
     }
     _op_save(t, e) {
         e = this.getBinData(e, t);
@@ -93,24 +83,22 @@ class e extends t {
                 fileHandle: i,
                 totalSavedSize: r,
                 filePath: s,
-                options: n
+                options: o
             } = t, {
-                emitSaveEvent: o,
+                emitSaveEvent: n,
                 maxFileSize: l,
                 maxRetryInterval: f,
-                retryIntervalFactor: h
-            } = n, p = S(a), u = p.length, c = `size ${v(u)} (MaxId:${e}) to ` + s, d;
+                retryIntervalFactor: p
+            } = o, h = S(a), u = h.length, d = `size ${v(u)} (MaxId:${e}) to ` + s, c;
             for (;"_op_truncate" !== t.firstOp?.opFunName; ) try {
-                await i.appendFile(p, {
-                    flush: !0
-                });
+                await i.write(h, 0, h.byteLength, r);
                 var m = t.totalSavedSize = r + u;
-                return o && 0 < e && t.emit("save", e), void (!t.maxFileSizeEmitted && l && m > l && (t.maxFileSizeEmitted = !0, 
+                return n && 0 < e && t.emit("save", e), void (!t.maxFileSizeEmitted && l && m > l && (t.maxFileSizeEmitted = !0, 
                 t.emit("maxFileSize", m)));
             } catch (t) {
-                g.error(`Saving ${c} failed:`, t, "Truncated back to " + r), await i.truncate(r), 
-                d = d ? Math.min(f, d * h) : 1, g.log(`Retry in ${d} seconds`), 
-                await _(d);
+                g.error(`Saving ${d} failed:`, t, "Truncated back to " + r), await i.truncate(r), 
+                c = c ? Math.min(f, c * p) : 1, g.log(`Retry in ${c} seconds`), 
+                await _(c);
             }
         })(this, t, e);
     }
@@ -124,7 +112,7 @@ class e extends t {
         } catch (t) {
             g.error(`Truncate ${e} failed:`, t);
         }
-        f(this);
+        h(this);
     }
     async _op_switch(t) {
         var {
@@ -140,17 +128,20 @@ async function i(t, e) {
         fileHandle: a,
         options: i
     } = t;
-    f(t);
+    h(t);
     try {
-        a && (a.close(), t.fileHandle = void 0), await n(e);
-        var r = i.append;
-        t.fileHandle = await s(e, r ? "a" : "w"), t.filePath = e, r && (t.totalSavedSize = await o(e));
+        a && (a.close(), t.fileHandle = void 0), await l(e), t.filePath = e;
+        var {
+            append: r,
+            read: s
+        } = i, o = r && await p(e) ? "r+" : s ? "w+" : "w";
+        t.fileHandle = await n(e, o), r && (t.totalSavedSize = await f(e));
     } catch (t) {
         g.error(`Open ${e} failed:`, t);
     }
 }
 
-function f(t) {
+function h(t) {
     Object.assign(t, {
         totalSavedSize: 0,
         maxFileSizeEmitted: !1

package/src/QyCD.d.ts

@@ -14,7 +14,7 @@ export declare class QyCD {
     file(namePath: string | string[], parentDir?: any): QyFile | undefined;
     fileP(namePath: string | string[], parentDir?: any): QyFile | undefined;
     fileContent(namePath: string | string[], parentDir?: any): any;
-    embeddingFile(namePath: string | string[], parentDir?: any): any;
+    infoFile(namePath: string | string[], parentDir?: any): any;
     embedding(namePath: string | string[], parentDir?: any): any;
     triggerFile(triggerName: any): any;
     get triggerFileList(): any;

package/src/QyCD.js

@@ -1,118 +1,120 @@
 import {
-    isString as t,
+    isString as i,
     pathSplit as n
 } from "./QyUtils.js";
 
-class e {
+class r {
     qyCache;
     currentDir;
-    constructor(e) {
-        this.qyCache = e;
+    constructor(r) {
+        this.qyCache = r;
     }
-    cd(e, r = this.currentDir) {
-        return t(e) && (e = n(e)), this.cdP(e, r);
+    cd(r, e = this.currentDir) {
+        return i(r) && (r = n(r)), this.cdP(r, e);
     }
-    cdP(e, r = this.currentDir) {
-        return this.currentDir = this.qyCache.getOrCreateDir(e, r), this;
+    cdP(r, e = this.currentDir) {
+        return this.currentDir = this.qyCache.getOrCreateDir(r, e), this;
     }
-    dir(e, r = this.currentDir) {
-        return t(e) && (e = n(e)), this.dirP(e, r);
+    dir(r, e = this.currentDir) {
+        return i(r) && (r = n(r)), this.dirP(r, e);
     }
-    dirP(e, r = this.currentDir) {
-        return this.qyCache.getDir(e, r);
+    dirP(r, e = this.currentDir) {
+        return this.qyCache.getDir(r, e);
     }
-    indexDir(e, r, i = this.currentDir) {
-        return t(e) && (e = n(e)), t(r) && (r = n(r)), this.indexDirP(e, r, i);
+    indexDir(r, e, t = this.currentDir) {
+        return i(r) && (r = n(r)), i(e) && (e = n(e)), this.indexDirP(r, e, t);
     }
-    indexDirP(e, r, i = this.currentDir) {
-        var t = this.qyCache, e = t.getDir(e, i);
-        if (e) {
-            i = t.getIndexDir(e);
-            if (i) return r && 0 != r.length ? t.getDir(r, i) : i;
+    indexDirP(r, e, t = this.currentDir) {
+        var i = this.qyCache, r = i.getDir(r, t);
+        if (r) {
+            t = i.getIndexDir(r);
+            if (t) return e && 0 != e.length ? i.getDir(e, t) : t;
         }
     }
-    file(e, r = this.currentDir) {
-        return t(e) && (e = n(e)), this.fileP(e, r);
+    file(r, e = this.currentDir) {
+        return i(r) && (r = n(r)), this.fileP(r, e);
     }
-    fileP(e, r = this.currentDir) {
-        return this.qyCache.getFile(e, r);
+    fileP(r, e = this.currentDir) {
+        return this.qyCache.getFile(r, e);
     }
-    fileContent(e, r = this.currentDir) {
-        return this.file(e, r)?.fileContent;
+    fileContent(r, e = this.currentDir) {
+        return this.file(r, e)?.fileContent;
     }
-    embeddingFile(e, r = this.currentDir) {
-        t(e) && (e = n(e));
-        var i = this.qyCache;
-        return i.getEmbeddingFile(i.getOrCreateFile(e, r).fullPathHash);
+    infoFile(r, e = this.currentDir) {
+        i(r) && (r = n(r));
+        var t = this.qyCache;
+        return t.getInfoFile(t.getOrCreateFile(r, e));
     }
-    embedding(e, r = this.currentDir) {
-        return this.embeddingFile(e, r)?.fileContent;
+    embedding(r, e = this.currentDir) {
+        i(r) && (r = n(r));
+        var t = this.qyCache, r = t.getOrCreateFile(r, e);
+        return t.getEmbedding(r);
     }
-    triggerFile(e) {
-        return this.qyCache.getTriggerFile(e);
+    triggerFile(r) {
+        return this.qyCache.getTriggerFile(r);
     }
     get triggerFileList() {
         return this.qyCache.triggerFileList;
     }
-    rpcFile(e) {
-        return this.qyCache.getRpcFile(e);
+    rpcFile(r) {
+        return this.qyCache.getRpcFile(r);
     }
     get rpcFileList() {
         return this.qyCache.rpcFileList;
     }
-    view(e, r = void 0, i = this.currentDir) {
-        return t(e) && (e = n(e)), this.viewP(e, r, i);
+    view(r, e = void 0, t = this.currentDir) {
+        return i(r) && (r = n(r)), this.viewP(r, e, t);
     }
-    viewP(e, r, i = this.currentDir) {
-        return this.qyCache.getFile(e, i)?.view(r);
+    viewP(r, e, t = this.currentDir) {
+        return this.qyCache.getFile(r, t)?.view(e);
     }
-    queryFiles(e, r, i = void 0, t = void 0, n = this.currentDir) {
-        return this.qyCache.queryFiles(e, r, i, t, n);
+    queryFiles(r, e, t = void 0, i = void 0, n = this.currentDir) {
+        return this.qyCache.queryFiles(r, e, t, i, n);
     }
-    queryFilesMulti(e, r, i, t = this.currentDir) {
-        return this.qyCache.queryFilesMulti(e, r, i, t);
+    queryFilesMulti(r, e, t, i = this.currentDir) {
+        return this.qyCache.queryFilesMulti(r, e, t, i);
     }
-    queryTree(e, r, i, t, n, s = this.currentDir) {
-        return h(this.qyCache.queryFiles(e, r, t, n, s), i);
+    queryTree(r, e, t, i, n, s = this.currentDir) {
+        return h(this.qyCache.queryFiles(r, e, i, n, s), t);
     }
-    queryTreeMulti(e, r, i, t, n = this.currentDir) {
-        return h(this.qyCache.queryFilesMulti(e, i, t, n), r);
+    queryTreeMulti(r, e, t, i, n = this.currentDir) {
+        return h(this.qyCache.queryFilesMulti(r, t, i, n), e);
     }
     resetCurrentDir() {
         this.currentDir = void 0;
     }
-    fileListToTree(e, r) {
-        return h(e, r);
+    fileListToTree(r, e) {
+        return h(r, e);
     }
 }
 
-function h(e, i) {
-    var t, n = {};
-    for (t of e) {
-        var s, h = t.parentList;
-        let r = n;
+function h(r, t) {
+    var i, n = {};
+    for (i of r) {
+        var s, h = i.parentList;
+        let e = n;
         for (s of h) {
             var {
                 name: u,
-                fullPathHash: l
+                fullPathHash: c
             } = s;
-            let e = r.subdirMap;
-            e = e || (r.subdirMap = {}), r = (r = e[u]) || (e[u] = {
-                fullPathHash: l
+            let r = e.subdirMap;
+            r = r || (e.subdirMap = {}), e = (e = r[u]) || (r[u] = {
+                fullPathHash: c
             });
         }
-        let e = r.fileMap;
-        (e = e || (r.fileMap = {}))[t.name] = {
-            fileContent: t.view(i, !0),
-            fileContentKey: t.fileContentKey
+        let r = e.fileMap;
+        (r = r || (e.fileMap = {}))[i.name] = {
+            fileContent: i.view(t, !0),
+            fileContentKey: i.fileContentKey
         };
     }
     return {
         tree: n,
-        count: e.length
+        count: r.length
     };
 }
 
 export {
-    e as QyCD
+    r as QyCD
 };

package/src/QyCache.d.ts

@@ -12,11 +12,13 @@ export declare class QyCache extends EventEmitter {
     rpcRootDir: any;
     indexRootDir: any;
     expireRootDir: any;
-    embeddingRootDir: any;
-    h2FRootDir: any;
+    hnswRootDir: any;
+    infoRootDir: any;
+    recycleRootDir: any;
     exclusiveFilePath: any;
     qyKVData: any;
     qyFileLogger: any;
+    qyEmbeddings: any;
     rpcFunMap: {};
     constructor(rootFolder: string, options?: any, qyDB?: any);
     start(): Promise<this>;
@@ -25,8 +27,8 @@ export declare class QyCache extends EventEmitter {
     decreaseChangeId(): void;
     save(changeId: any, cmdArgAsListObj: any, logText: any, syncList: any, needPromise: any): any;
     getFile(namePath: QyFile | string | string[], parentDir?: any): QyFile | undefined;
-    getDir(namePath: QyDir | string | string[], parentDir?: any): QyDir | undefined;
     getOrCreateFile(namePath: QyFile | string | string[], parentDir?: any): QyFile;
+    getDir(namePath: QyDir | string | string[], parentDir?: any): QyDir | undefined;
     getOrCreateDir(namePath: QyDir | string | string[], parentDir?: any): QyDir;
     getIndexDir(indexHostDir: QyDir): any;
     getOrCreateIndexDir(indexHostDir: any): any;
@@ -34,12 +36,12 @@ export declare class QyCache extends EventEmitter {
     getOrCreateTriggerFile(triggerName: any): any;
     getExpireInfoFile(fullPathHash: any, type?: string): any;
     getOrCreateExpireInfoFile(fullPathHash: any, type?: string): any;
-    getH2FFile(fullPathHash: any): any;
-    getOrCreateH2FFile(fullPathHash: any): any;
-    h2F(fullPathHash: any): QyFile | undefined;
-    getEmbeddingFile(fullPathHash: any): any;
-    getOrCreateEmbeddingFile(fullPathHash: any): any;
-    getEmbedding(fullPathHash: any): any;
+    getInfoFile(srcFile: any): any;
+    getOrCreateInfoFile(srcFile: any): any;
+    hToSrcFile(fullPathHash: any): QyFile | undefined;
+    getEmbedding(srcFile: any): any;
+    getOrCreateRecycleFile(length: any): any;
+    getRecycleFile(length: any): any;
     get triggerFileList(): any;
     getRpcFile(rpcFunName: any): any;
     getOrCreateRpcFile(rpcFunName: any): any;