datafolder 1.0.18 → 1.0.19
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/package.json +1 -1
- package/sample.js +9 -0
- package/scripts/fastify.js +1 -0
- package/scripts/hnsw.js +39 -0
- package/src/QyAcl.js +1 -1
- package/src/QyBatch.d.ts +2 -1
- package/src/QyBatch.js +142 -101
- package/src/QyBinWriter.d.ts +5 -9
- package/src/QyBinWriter.js +36 -45
- package/src/QyCD.d.ts +1 -1
- package/src/QyCD.js +66 -64
- package/src/QyCache.d.ts +11 -9
- package/src/QyCache.js +125 -121
- package/src/QyDB.js +1 -4
- package/src/QyDefaultOptions.js +8 -0
- package/src/QyEmbeddingWriter.d.ts +12 -0
- package/src/QyEmbeddingWriter.js +55 -0
- package/src/QyEmbeddings.d.ts +15 -0
- package/src/QyEmbeddings.js +128 -0
- package/src/QyFileContentUtils.js +123 -111
- package/src/QyHNSW.d.ts +5 -8
- package/src/QyHNSW.js +36 -44
- package/src/QySnapshots.js +177 -184
- package/src/QyUtils.d.ts +5 -3
- package/src/QyUtils.js +130 -114
- package/static/codemirror.bundle/editor.bundle.js +16 -9
package/README.md
CHANGED
|
@@ -4,6 +4,8 @@ datafolder is a lightning fluid JSON database for high speed SSDs, featuring an
|
|
|
4
4
|
|
|
5
5
|
### Changelog
|
|
6
6
|
|
|
7
|
+
1.0.19 Add DocumentDB test case (test/testQyDocumentDB.js)
|
|
8
|
+
|
|
7
9
|
1.0.18 Add Embedding Storage Support. Related APIs: dataFolder.insertEmbedding() and dataFolder.delEmbedding(). TODO: Add HNSW support and vector query.
|
|
8
10
|
|
|
9
11
|
1.0.17 Add Northwind sample data test (test/testQyNorthwind.js)
|
package/package.json
CHANGED
package/sample.js
CHANGED
|
@@ -14,6 +14,11 @@ async function run() {
|
|
|
14
14
|
dataFolder.insert("traindata/2026/1/20/chat_0001/prompt", "What is async/await?");
|
|
15
15
|
dataFolder.insert("traindata/2026/1/20/chat_0001/answer", "async/await is ...");
|
|
16
16
|
dataFolder.insertEmbedding("traindata/2026/1/20/chat_0001/prompt", [235.3536, 19.2358, 92.0929846]);
|
|
17
|
+
dataFolder.insertWithEmbedding(
|
|
18
|
+
"traindata/content",
|
|
19
|
+
{ score: 90, prompt: "hello" },
|
|
20
|
+
[235.3536, 19.2358, 92.0929846],
|
|
21
|
+
);
|
|
17
22
|
const { userid, email, address, feedbackScore } = dataFolder.view("traindata/2026/1/20/chat_0001");
|
|
18
23
|
logger.log(userid, email, feedbackScore, address);
|
|
19
24
|
address.zipcode = 678;
|
|
@@ -22,6 +27,8 @@ async function run() {
|
|
|
22
27
|
logger.debug(dataFolder.view("traindata/2026/1/20/chat_0001/prompt"));
|
|
23
28
|
logger.warn(dataFolder.view("traindata/2026/1/20/chat_0001/answer"));
|
|
24
29
|
logger.log(dataFolder.embedding("traindata/2026/1/20/chat_0001/prompt"));
|
|
30
|
+
logger.log(dataFolder.view("traindata/content"));
|
|
31
|
+
logger.log(dataFolder.embedding("traindata/content"));
|
|
25
32
|
await dataFolder.stop();
|
|
26
33
|
await dataFolder.start();
|
|
27
34
|
{
|
|
@@ -35,6 +42,8 @@ async function run() {
|
|
|
35
42
|
dataFolder.view("traindata/2026/1/20/chat_0001/answer"),
|
|
36
43
|
);
|
|
37
44
|
logger.log(dataFolder.embedding("traindata/2026/1/20/chat_0001/prompt"));
|
|
45
|
+
logger.log(dataFolder.view("traindata/content"));
|
|
46
|
+
logger.log(dataFolder.embedding("traindata/content"));
|
|
38
47
|
}
|
|
39
48
|
await dataFolder.stop();
|
|
40
49
|
}
|
package/scripts/fastify.js
CHANGED
|
@@ -15,6 +15,7 @@ const config = {
|
|
|
15
15
|
const { port, staticRoot } = config;
|
|
16
16
|
const fastify = Fastify({ logger: false, forceCloseConnections: true });
|
|
17
17
|
// const qyDB = new QyDB(join(__dirname, "../tmp/simulate2"));
|
|
18
|
+
// const qyDB = new QyDB(join(__dirname, "../tmp/QyEmbeddings/1"));
|
|
18
19
|
const qyDB = new QyDB();
|
|
19
20
|
let sigintReceived = false;
|
|
20
21
|
process.on("SIGINT", async () => {
|
package/scripts/hnsw.js
ADDED
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
import { join, basename } from "node:path";
|
|
3
|
+
import { readdir, readFile } from "node:fs/promises";
|
|
4
|
+
import fastifyStatic from "@fastify/static";
|
|
5
|
+
import fastifyCompress from "@fastify/compress";
|
|
6
|
+
import { QyDB } from "../src/QyDB.js";
|
|
7
|
+
import { folderOrFileExists, __dirname } from "../src/QyUtils.js";
|
|
8
|
+
import { logger } from "../src/QyLogger.js";
|
|
9
|
+
// logger.level = "warn";
|
|
10
|
+
const baseVectPath = "D:\\syncsoft\\hnsw\\bench\\datasets\\siftsmall\\siftsmall_base.fvecs";
|
|
11
|
+
const qyDB = new QyDB();
|
|
12
|
+
run();
|
|
13
|
+
async function run() {
|
|
14
|
+
await qyDB.start({ appendOnly: false });
|
|
15
|
+
const vectorList = await loadVects(baseVectPath);
|
|
16
|
+
const { length } = vectorList;
|
|
17
|
+
console.time(`insertEmbedding${length}`);
|
|
18
|
+
for (let idx = 0; idx < length; ++idx) {
|
|
19
|
+
const vect = vectorList[idx];
|
|
20
|
+
qyDB.insertEmbedding(["embedding", idx], vect);
|
|
21
|
+
// qyDB.insert(["prompt", idx], new Uint8Array(vect.buffer, vect.byteOffset, vect.byteLength));
|
|
22
|
+
// qyDB.delEmbedding(["embedding", idx]);
|
|
23
|
+
// qyDB.delFile(["prompt", idx]);
|
|
24
|
+
}
|
|
25
|
+
await qyDB.stop();
|
|
26
|
+
console.timeEnd(`insertEmbedding${length}`);
|
|
27
|
+
}
|
|
28
|
+
async function loadVects(filePath) {
|
|
29
|
+
const buffer = await readFile(filePath);
|
|
30
|
+
const { length } = buffer;
|
|
31
|
+
let offset = 0;
|
|
32
|
+
const vectorList = [];
|
|
33
|
+
while (offset < length) {
|
|
34
|
+
const dim = buffer.readUint32LE(offset);
|
|
35
|
+
vectorList.push(new Float32Array(buffer.buffer, offset + 4, dim));
|
|
36
|
+
offset += (dim + 1) * 4;
|
|
37
|
+
}
|
|
38
|
+
return vectorList;
|
|
39
|
+
}
|
package/src/QyAcl.js
CHANGED
package/src/QyBatch.d.ts
CHANGED
|
@@ -27,6 +27,7 @@ export declare class QyBatch extends QyCD {
|
|
|
27
27
|
insert(namePath: any, content: any, noClone?: boolean, noCreate?: boolean, parentDir?: any): any;
|
|
28
28
|
insertEmbedding(namePath: any, embedding: any, parentDir?: any): any;
|
|
29
29
|
delEmbedding(namePath: any, parentDir?: any): any;
|
|
30
|
+
insertWithEmbedding(namePath: any, content: any, embedding: any, noClone?: boolean, noCreate?: boolean, parentDir?: any): any;
|
|
30
31
|
remove(namePath: any, content: any, parentDir?: any): any;
|
|
31
32
|
put(namePath: any, content: any, noClone?: boolean, parentDir?: any): any;
|
|
32
33
|
delContent(namePath: any, parentDir?: any): any;
|
|
@@ -46,7 +47,7 @@ export declare class QyBatch extends QyCD {
|
|
|
46
47
|
_action_delDir(action: any): boolean;
|
|
47
48
|
_action_delContent(action: any): boolean;
|
|
48
49
|
_action_insertEmbedding(action: any): boolean;
|
|
49
|
-
_action_delEmbedding(action: any):
|
|
50
|
+
_action_delEmbedding(action: any): true | undefined;
|
|
50
51
|
_action_insert(action: any): boolean;
|
|
51
52
|
_action_createFile(action: any): boolean;
|
|
52
53
|
_action_createDir(action: any): boolean;
|
package/src/QyBatch.js
CHANGED
|
@@ -1,27 +1,27 @@
|
|
|
1
1
|
import {
|
|
2
|
-
isDeepStrictEqual as
|
|
2
|
+
isDeepStrictEqual as c
|
|
3
3
|
} from "node:util";
|
|
4
4
|
|
|
5
5
|
import {
|
|
6
|
-
mergeContentAndGenDelta as
|
|
6
|
+
mergeContentAndGenDelta as o,
|
|
7
7
|
objToDelMarks as r
|
|
8
8
|
} from "./QyFileContentUtils.js";
|
|
9
9
|
|
|
10
10
|
import {
|
|
11
11
|
pathSplit as s,
|
|
12
12
|
ensurePropNamePathList as n,
|
|
13
|
-
isString as
|
|
13
|
+
isString as l,
|
|
14
14
|
SpecialOperators as e,
|
|
15
|
-
getSuffix as
|
|
15
|
+
getSuffix as f,
|
|
16
16
|
SpecialFilePaths as t,
|
|
17
|
-
toSingleName as
|
|
17
|
+
toSingleName as C,
|
|
18
18
|
isSimpleType as h,
|
|
19
19
|
isEmptyObj as d,
|
|
20
20
|
isNumber as a,
|
|
21
21
|
getNowRounded as m,
|
|
22
22
|
isDate as u,
|
|
23
|
-
isArray as
|
|
24
|
-
isFloat32Array as
|
|
23
|
+
isArray as g,
|
|
24
|
+
isFloat32Array as p
|
|
25
25
|
} from "./QyUtils.js";
|
|
26
26
|
|
|
27
27
|
import {
|
|
@@ -49,7 +49,7 @@ import {
|
|
|
49
49
|
logger as F
|
|
50
50
|
} from "./QyLogger.js";
|
|
51
51
|
|
|
52
|
-
let O = e.$del,
|
|
52
|
+
let O = e.$del, b = t.IndexPropPathsFileName, L = {
|
|
53
53
|
none: 0,
|
|
54
54
|
basic: 1,
|
|
55
55
|
delta: 2,
|
|
@@ -70,28 +70,36 @@ class Q extends D {
|
|
|
70
70
|
..._("QyBatch"),
|
|
71
71
|
...t
|
|
72
72
|
};
|
|
73
|
-
e =
|
|
73
|
+
e = L[t.fileLogLevel.toLowerCase()];
|
|
74
74
|
Object.assign(this, {
|
|
75
75
|
options: t,
|
|
76
76
|
fileLogLevel: e ?? 1,
|
|
77
77
|
qyAclCmdGenerator: new i(),
|
|
78
|
-
qyCmdObjHistory: new
|
|
78
|
+
qyCmdObjHistory: new A(t.maxHistoryCmdObjCount),
|
|
79
79
|
actionQueue: [],
|
|
80
80
|
changeList: [],
|
|
81
81
|
syncList: []
|
|
82
82
|
});
|
|
83
83
|
}
|
|
84
84
|
createDir(e, t = this.currentDir) {
|
|
85
|
-
return
|
|
85
|
+
return l(e) && (e = s(e)), q(this, "createDir", {
|
|
86
|
+
item: this.qyCache.getOrCreateDir(e, t)
|
|
87
|
+
});
|
|
86
88
|
}
|
|
87
89
|
delDir(e, t = this.currentDir) {
|
|
88
|
-
return
|
|
90
|
+
return l(e) && (e = s(e)), q(this, "delDir", {
|
|
91
|
+
item: this.qyCache.getOrCreateDir(e, t).ensureAllLoaded()
|
|
92
|
+
});
|
|
89
93
|
}
|
|
90
94
|
createFile(e, t = this.currentDir) {
|
|
91
|
-
return
|
|
95
|
+
return l(e) && (e = s(e)), q(this, "createFile", {
|
|
96
|
+
item: this.qyCache.getOrCreateFile(e, t).ensureLoaded()
|
|
97
|
+
});
|
|
92
98
|
}
|
|
93
99
|
delFile(e, t = this.currentDir) {
|
|
94
|
-
return
|
|
100
|
+
return l(e) && (e = s(e)), q(this, "delFile", {
|
|
101
|
+
item: this.qyCache.getOrCreateFile(e, t).ensureLoaded()
|
|
102
|
+
});
|
|
95
103
|
}
|
|
96
104
|
expireFile(e, t, i = this.currentDir) {
|
|
97
105
|
return this.expireFileM(e, 1e3 * t, i);
|
|
@@ -106,9 +114,10 @@ class Q extends D {
|
|
|
106
114
|
return T(this, e, t, "d", i);
|
|
107
115
|
}
|
|
108
116
|
expireContent(e, t, i, r = this.currentDir) {
|
|
109
|
-
|
|
117
|
+
l(e) && (e = s(e));
|
|
110
118
|
e = this.qyCache.getOrCreateFile(e, r).ensureLoaded();
|
|
111
|
-
return
|
|
119
|
+
return q(this, "expireContent", {
|
|
120
|
+
item: e,
|
|
112
121
|
expireInfoObj: t,
|
|
113
122
|
isMilliSecond: i
|
|
114
123
|
});
|
|
@@ -123,44 +132,56 @@ class Q extends D {
|
|
|
123
132
|
return M(this, e, "d", t, i);
|
|
124
133
|
}
|
|
125
134
|
removeContentExpire(e, t, i = !1, r = this.currentDir) {
|
|
126
|
-
|
|
135
|
+
l(e) && (e = s(e));
|
|
127
136
|
e = this.qyCache.getOrCreateFile(e, r).ensureLoaded();
|
|
128
|
-
return
|
|
137
|
+
return q(this, "removeContentExpire", {
|
|
138
|
+
item: e,
|
|
129
139
|
expireInfoObj: t,
|
|
130
140
|
infoFileOnly: i
|
|
131
141
|
});
|
|
132
142
|
}
|
|
133
143
|
createIndex(e, t, i = this.currentDir) {
|
|
134
|
-
|
|
144
|
+
l(e) && (e = s(e)), g(t) || (t = [ t ]);
|
|
145
|
+
e = this.qyCache.getOrCreateDir(e, i);
|
|
146
|
+
return q(this, "createIndex", {
|
|
147
|
+
item: e,
|
|
135
148
|
indexPropPathList: t
|
|
136
149
|
});
|
|
137
150
|
}
|
|
138
151
|
delIndex(e, t, i = this.currentDir) {
|
|
139
|
-
|
|
152
|
+
l(e) && (e = s(e));
|
|
153
|
+
e = this.qyCache.getOrCreateDir(e, i);
|
|
154
|
+
return q(this, "delIndex", {
|
|
155
|
+
item: e,
|
|
140
156
|
indexPropPath: t
|
|
141
157
|
});
|
|
142
158
|
}
|
|
143
159
|
insert(e, t, i = !1, r = !1, n = this.currentDir) {
|
|
144
|
-
|
|
160
|
+
l(e) && (e = s(e));
|
|
145
161
|
e = this.qyCache.getOrCreateFile(e, n).ensureLoaded();
|
|
146
|
-
return
|
|
147
|
-
|
|
162
|
+
return q(this, "insert", {
|
|
163
|
+
item: e,
|
|
164
|
+
content: t = i || h(t) ? t : structuredClone(t),
|
|
148
165
|
noCreate: r
|
|
149
166
|
});
|
|
150
167
|
}
|
|
151
168
|
insertEmbedding(e, t, i = this.currentDir) {
|
|
152
|
-
var r = this.qyCache,
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
169
|
+
var r = this.qyCache, r = (p(t) || (t = new Float32Array(t)), l(e) && (e = s(e)),
|
|
170
|
+
r.getOrCreateFile(e, i));
|
|
171
|
+
return q(this, "insertEmbedding", {
|
|
172
|
+
item: r,
|
|
173
|
+
embedding: t
|
|
156
174
|
});
|
|
157
175
|
}
|
|
158
176
|
delEmbedding(e, t = this.currentDir) {
|
|
159
|
-
var i = this.qyCache,
|
|
160
|
-
return
|
|
161
|
-
|
|
177
|
+
var i = this.qyCache, i = (l(e) && (e = s(e)), i.getOrCreateFile(e, t));
|
|
178
|
+
return q(this, "delEmbedding", {
|
|
179
|
+
item: i
|
|
162
180
|
});
|
|
163
181
|
}
|
|
182
|
+
insertWithEmbedding(e, t, i, r = !1, n = !1, a = this.currentDir) {
|
|
183
|
+
return this.insert(e, t, r, n, a), this.insertEmbedding(e, i, a);
|
|
184
|
+
}
|
|
164
185
|
remove(e, t, i = this.currentDir) {
|
|
165
186
|
return this.insert(e, r(t), !0, !0, i);
|
|
166
187
|
}
|
|
@@ -170,25 +191,38 @@ class Q extends D {
|
|
|
170
191
|
}, i, !1, r);
|
|
171
192
|
}
|
|
172
193
|
delContent(e, t = this.currentDir) {
|
|
173
|
-
|
|
194
|
+
l(e) && (e = s(e));
|
|
195
|
+
e = this.qyCache.getOrCreateFile(e, t).ensureLoaded();
|
|
196
|
+
return q(this, "delContent", {
|
|
197
|
+
item: e
|
|
198
|
+
});
|
|
174
199
|
}
|
|
175
200
|
insertTrigger(e, t, i, r) {
|
|
176
|
-
|
|
201
|
+
l(t) && (t = s(t)), i = n(i);
|
|
202
|
+
e = this.qyCache.getOrCreateTriggerFile(e);
|
|
203
|
+
return q(this, "insertTrigger", {
|
|
204
|
+
item: e,
|
|
177
205
|
fileNamePath: t,
|
|
178
206
|
propNamePathList: i,
|
|
179
207
|
modCode: r
|
|
180
208
|
});
|
|
181
209
|
}
|
|
182
210
|
removeTrigger(e) {
|
|
183
|
-
return
|
|
211
|
+
return q(this, "removeTrigger", {
|
|
212
|
+
item: this.qyCache.getOrCreateTriggerFile(e)
|
|
213
|
+
});
|
|
184
214
|
}
|
|
185
215
|
insertRpc(e, t) {
|
|
186
|
-
|
|
216
|
+
e = this.qyCache.getOrCreateRpcFile(e);
|
|
217
|
+
return q(this, "insertRpc", {
|
|
218
|
+
item: e,
|
|
187
219
|
modCode: t
|
|
188
220
|
});
|
|
189
221
|
}
|
|
190
222
|
removeRpc(e) {
|
|
191
|
-
return
|
|
223
|
+
return q(this, "removeRpc", {
|
|
224
|
+
item: this.qyCache.getOrCreateRpcFile(e)
|
|
225
|
+
});
|
|
192
226
|
}
|
|
193
227
|
run(e = !1) {
|
|
194
228
|
var {
|
|
@@ -209,7 +243,7 @@ class Q extends D {
|
|
|
209
243
|
syncList: l
|
|
210
244
|
} = this, c = i.length;
|
|
211
245
|
if (0 == c) return n.decreaseChangeId(), 0;
|
|
212
|
-
var h =
|
|
246
|
+
var h = f(c), s = s.takeCmdArgAsListObj();
|
|
213
247
|
let d;
|
|
214
248
|
0 < r && (r = i.map(({
|
|
215
249
|
logStr: e
|
|
@@ -232,7 +266,7 @@ ${r}
|
|
|
232
266
|
for ({
|
|
233
267
|
item: m,
|
|
234
268
|
delta: u
|
|
235
|
-
} of h) for (var
|
|
269
|
+
} of h) for (var g of m.triggerNodes) g.checkTrigger(m, u);
|
|
236
270
|
return e ? c : t;
|
|
237
271
|
}
|
|
238
272
|
getTopHistoryCmdObj() {
|
|
@@ -252,7 +286,7 @@ ${r}
|
|
|
252
286
|
item: e,
|
|
253
287
|
content: [ i, r, n ]
|
|
254
288
|
});
|
|
255
|
-
return e && (t = this.qyCache, s && !
|
|
289
|
+
return e && (t = this.qyCache, s && !c(s, i) && t.removeTriggerFromTree(a, s),
|
|
256
290
|
t.insertTriggerToTree(a, i, r, n)), e;
|
|
257
291
|
}
|
|
258
292
|
_action_removeTrigger(e) {
|
|
@@ -288,14 +322,14 @@ ${r}
|
|
|
288
322
|
var e, {
|
|
289
323
|
item: t,
|
|
290
324
|
indexPropPathList: i
|
|
291
|
-
} = t, r = this.qyCache.getOrCreateIndexDir(t), n = r.getOrCreateFile(
|
|
325
|
+
} = t, r = this.qyCache.getOrCreateIndexDir(t), n = r.getOrCreateFile(b), {
|
|
292
326
|
fileContent: a,
|
|
293
327
|
qyIndexPropTree: s
|
|
294
328
|
} = (n.created || this._action_createFile({
|
|
295
329
|
item: n
|
|
296
330
|
}), n), o = [], l = {};
|
|
297
331
|
for (e of i) {
|
|
298
|
-
var c =
|
|
332
|
+
var c = C(e);
|
|
299
333
|
a && a[c] || (o.push(e), l[c] = 1, s.insert(e));
|
|
300
334
|
}
|
|
301
335
|
if (0 != o.length) {
|
|
@@ -307,17 +341,17 @@ ${r}
|
|
|
307
341
|
var h, d = this, m = t, u = r, i = x(o);
|
|
308
342
|
let e;
|
|
309
343
|
for (h of i) {
|
|
310
|
-
var
|
|
311
|
-
if (!
|
|
344
|
+
var g = u.searchOrCreateSubdir(h);
|
|
345
|
+
if (!g.created) {
|
|
312
346
|
d._action_createDir({
|
|
313
|
-
item:
|
|
347
|
+
item: g
|
|
314
348
|
});
|
|
315
|
-
for (var
|
|
349
|
+
for (var f of e = e || m.fileList) {
|
|
316
350
|
var {
|
|
317
|
-
fileContent:
|
|
318
|
-
name:
|
|
319
|
-
} =
|
|
320
|
-
null !=
|
|
351
|
+
fileContent: f,
|
|
352
|
+
name: p
|
|
353
|
+
} = f;
|
|
354
|
+
null != f && E(d, g, h, p, void 0, f);
|
|
321
355
|
}
|
|
322
356
|
}
|
|
323
357
|
}
|
|
@@ -330,10 +364,10 @@ ${r}
|
|
|
330
364
|
item: e,
|
|
331
365
|
indexPropPath: r
|
|
332
366
|
} = e, e = this.qyCache.getIndexDir(e);
|
|
333
|
-
e && ((i = e.getFile(
|
|
367
|
+
e && ((i = e.getFile(b)) ? (t = i.qyIndexPropTree, this._action_insert({
|
|
334
368
|
item: i,
|
|
335
369
|
content: {
|
|
336
|
-
[
|
|
370
|
+
[C(r)]: O
|
|
337
371
|
}
|
|
338
372
|
}), t.remove(r).hasPrefix || (i = e.getSubdir(r)) && this._action_delDir({
|
|
339
373
|
item: i
|
|
@@ -352,7 +386,7 @@ ${r}
|
|
|
352
386
|
parentDir: r,
|
|
353
387
|
name: n
|
|
354
388
|
} = i;
|
|
355
|
-
return i.created = !1,
|
|
389
|
+
return i.created = !1, N(this, e), t || I(this, r, y.R, r.fileMapKey, n),
|
|
356
390
|
!0;
|
|
357
391
|
}
|
|
358
392
|
_action_delDir(e) {
|
|
@@ -368,13 +402,13 @@ ${r}
|
|
|
368
402
|
fileMapKey: o
|
|
369
403
|
} = t;
|
|
370
404
|
if (a) {
|
|
371
|
-
|
|
405
|
+
I(this, t, y.D, o);
|
|
372
406
|
for (var l of t.fileList) this._action_delFile({
|
|
373
407
|
item: l
|
|
374
408
|
}, !0);
|
|
375
409
|
}
|
|
376
410
|
if (n) {
|
|
377
|
-
|
|
411
|
+
I(this, t, y.D, s);
|
|
378
412
|
for (var c of t.allSubdirs()) {
|
|
379
413
|
let {
|
|
380
414
|
subdirMap: e,
|
|
@@ -382,19 +416,19 @@ ${r}
|
|
|
382
416
|
subdirMapKey: i,
|
|
383
417
|
fileMapKey: r
|
|
384
418
|
} = c;
|
|
385
|
-
if (e &&
|
|
386
|
-
|
|
419
|
+
if (e && I(this, c, y.D, i), t) {
|
|
420
|
+
I(this, c, y.D, r);
|
|
387
421
|
for (var h of c.fileList) this._action_delFile({
|
|
388
422
|
item: h
|
|
389
423
|
}, !0);
|
|
390
424
|
}
|
|
391
|
-
|
|
425
|
+
N(this, {
|
|
392
426
|
actionName: "delDir",
|
|
393
427
|
item: c
|
|
394
428
|
}), c.created = !1;
|
|
395
429
|
}
|
|
396
430
|
}
|
|
397
|
-
return !!i && (t.created = !1,
|
|
431
|
+
return !!i && (t.created = !1, N(this, e), I(this, i, y.R, i.subdirMapKey, r),
|
|
398
432
|
!0);
|
|
399
433
|
}
|
|
400
434
|
_action_delContent(e) {
|
|
@@ -406,34 +440,42 @@ ${r}
|
|
|
406
440
|
oldValue: i,
|
|
407
441
|
newValue: O
|
|
408
442
|
}
|
|
409
|
-
}),
|
|
443
|
+
}), N(this, e), I(this, t, y.D, t.fileContentKey), P(this, t, i), !0);
|
|
410
444
|
}
|
|
411
445
|
_action_insertEmbedding(e) {
|
|
412
446
|
e.actionName = "insertEmbedding";
|
|
413
|
-
var {
|
|
414
|
-
item: e,
|
|
415
|
-
content: t,
|
|
416
|
-
srcFile: i
|
|
417
|
-
} = e, i = (this._action_insert({
|
|
418
|
-
item: this.qyCache.getOrCreateH2FFile(i.fullPathHash),
|
|
419
|
-
content: i.namePath
|
|
420
|
-
}), this._action_insert({
|
|
447
|
+
var t, i, r = this.qyCache, n = r.qyEmbeddings, {
|
|
421
448
|
item: e,
|
|
422
|
-
|
|
423
|
-
})
|
|
424
|
-
|
|
449
|
+
embedding: a
|
|
450
|
+
} = e, s = a.length, r = r.getOrCreateInfoFile(e), o = r.fileContent;
|
|
451
|
+
let l = !1;
|
|
452
|
+
return r.created && o.length == s ? (t = e.fullPathHash, o = o.idx, i = n.getEmbedding(t, s, o),
|
|
453
|
+
c(a, i) || ((i = n.updateEmbedding(t, a, o)) != o && this._action_insert({
|
|
454
|
+
item: r,
|
|
455
|
+
content: {
|
|
456
|
+
idx: i
|
|
457
|
+
}
|
|
458
|
+
}), l = !0)) : ({
|
|
459
|
+
fullPathHash: t,
|
|
460
|
+
namePath: o
|
|
461
|
+
} = e, i = n.addEmbedding(t, a), this._action_insert({
|
|
462
|
+
item: r,
|
|
463
|
+
content: {
|
|
464
|
+
length: s,
|
|
465
|
+
idx: i,
|
|
466
|
+
namePath: r.created ? void 0 : o
|
|
467
|
+
}
|
|
468
|
+
}), l = !0), l, l;
|
|
425
469
|
}
|
|
426
470
|
_action_delEmbedding(e) {
|
|
427
471
|
e.actionName = "delEmbedding";
|
|
428
|
-
var
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
item: this.qyCache.getOrCreateH2FFile(t.fullPathHash)
|
|
436
|
-
}), e;
|
|
472
|
+
var t = this.qyCache, i = t.qyEmbeddings, e = e.item;
|
|
473
|
+
let r, n;
|
|
474
|
+
var a, t = t.getInfoFile(e);
|
|
475
|
+
if (t && (a = t.fileContent, a) && (r = a.length, n = a.idx), i.delEmbedding(e.fullPathHash, r, n),
|
|
476
|
+
t) return this._action_delFile({
|
|
477
|
+
item: t
|
|
478
|
+
}), !0;
|
|
437
479
|
}
|
|
438
480
|
_action_insert(e) {
|
|
439
481
|
e.actionName = "insert";
|
|
@@ -450,15 +492,15 @@ ${r}
|
|
|
450
492
|
item: t
|
|
451
493
|
});
|
|
452
494
|
}
|
|
453
|
-
var n, a, r =
|
|
495
|
+
var n, a, r = o(t.fileContent, i), {
|
|
454
496
|
merged: i,
|
|
455
497
|
delta: s
|
|
456
498
|
} = r;
|
|
457
499
|
return !!s && ({
|
|
458
500
|
oldValue: s,
|
|
459
501
|
newValue: n
|
|
460
|
-
} = s, !!n) && (a = t.fileContentKey, null == i ?
|
|
461
|
-
Object.assign(e, r),
|
|
502
|
+
} = s, !!n) && (a = t.fileContentKey, null == i ? I(this, t, y.D, a) : I(this, t, y.M, a, n),
|
|
503
|
+
Object.assign(e, r), N(this, e), t.fileContent = i, P(this, t, s, n, i),
|
|
462
504
|
!0);
|
|
463
505
|
}
|
|
464
506
|
_action_createFile(e) {
|
|
@@ -469,7 +511,7 @@ ${r}
|
|
|
469
511
|
}), Object.assign(i, {
|
|
470
512
|
created: !0,
|
|
471
513
|
fileContentLoaded: !0
|
|
472
|
-
}),
|
|
514
|
+
}), N(this, e), I(this, t, y.A, t.fileMapKey, i.name), !0);
|
|
473
515
|
}
|
|
474
516
|
_action_createDir(e) {
|
|
475
517
|
e.actionName = "createDir";
|
|
@@ -481,12 +523,12 @@ ${r}
|
|
|
481
523
|
created: !0,
|
|
482
524
|
subdirMapLoaded: !0,
|
|
483
525
|
fileMapLoaded: !0
|
|
484
|
-
}),
|
|
526
|
+
}), N(this, {
|
|
485
527
|
actionName: "createDir",
|
|
486
528
|
item: e
|
|
487
529
|
});
|
|
488
530
|
var r = e.parentDir;
|
|
489
|
-
|
|
531
|
+
I(this, r, y.A, r.subdirMapKey, e.name);
|
|
490
532
|
}
|
|
491
533
|
return !0;
|
|
492
534
|
}
|
|
@@ -547,7 +589,7 @@ ${r}
|
|
|
547
589
|
expireInfoObj: i,
|
|
548
590
|
infoFileOnly: r
|
|
549
591
|
} = e, n = this.qyCache, a = n.getExpireInfoFile(e.fullPathHash, "c");
|
|
550
|
-
return !!a && (i =
|
|
592
|
+
return !!a && (i = g(i) ? Object.fromEntries(i.map(e => [ e, O ])) : j(i, e => e ? O : void 0),
|
|
551
593
|
(t = this._action_insert({
|
|
552
594
|
item: a,
|
|
553
595
|
content: i
|
|
@@ -557,15 +599,12 @@ ${r}
|
|
|
557
599
|
}
|
|
558
600
|
}
|
|
559
601
|
|
|
560
|
-
function
|
|
561
|
-
return e.actionQueue.push(
|
|
562
|
-
|
|
563
|
-
item: i,
|
|
564
|
-
...r
|
|
565
|
-
}), e;
|
|
602
|
+
function q(e, t, i) {
|
|
603
|
+
return e.running ? e["_action_" + t](i) : (i.actionName = t, e.actionQueue.push(i)),
|
|
604
|
+
e;
|
|
566
605
|
}
|
|
567
606
|
|
|
568
|
-
function
|
|
607
|
+
function N(e, t) {
|
|
569
608
|
var {
|
|
570
609
|
changeList: e,
|
|
571
610
|
fileLogLevel: i
|
|
@@ -589,7 +628,7 @@ function b(e, t) {
|
|
|
589
628
|
})(t, i)), e.push(t);
|
|
590
629
|
}
|
|
591
630
|
|
|
592
|
-
function
|
|
631
|
+
function I(e, t, i, r, n = void 0) {
|
|
593
632
|
var {
|
|
594
633
|
qyAclCmdGenerator: e,
|
|
595
634
|
qyCache: a,
|
|
@@ -600,7 +639,7 @@ function P(e, t, i, r, n = void 0) {
|
|
|
600
639
|
a && a.qyKVData.removeKey(r) && s.push(r, t);
|
|
601
640
|
}
|
|
602
641
|
|
|
603
|
-
function
|
|
642
|
+
function P(e, t, i, r = void 0, n = void 0) {
|
|
604
643
|
var {
|
|
605
644
|
underHiddenFolder: t,
|
|
606
645
|
name: a,
|
|
@@ -609,7 +648,7 @@ function I(e, t, i, r = void 0, n = void 0) {
|
|
|
609
648
|
if (!t) {
|
|
610
649
|
var o = e.qyCache.getIndexDir(s);
|
|
611
650
|
if (o) {
|
|
612
|
-
t = o.getFile(
|
|
651
|
+
t = o.getFile(b);
|
|
613
652
|
if (t) for (var l of t.qyIndexPropTree.getAllIndexPropPaths()) E(e, o.searchOrCreateSubdir(l), l, a, i, r, n);
|
|
614
653
|
}
|
|
615
654
|
}
|
|
@@ -637,7 +676,7 @@ function E(t, e, i, r, n, a, s = void 0) {
|
|
|
637
676
|
}
|
|
638
677
|
}
|
|
639
678
|
|
|
640
|
-
class
|
|
679
|
+
class A {
|
|
641
680
|
maxLength;
|
|
642
681
|
actionQueue;
|
|
643
682
|
nextPos;
|
|
@@ -677,19 +716,21 @@ function j(e, t = void 0, i = [], r = {}) {
|
|
|
677
716
|
}
|
|
678
717
|
|
|
679
718
|
function S(e, t, i) {
|
|
680
|
-
return a(e) ? (t ? e : 1e3 * e) + i :
|
|
719
|
+
return a(e) ? (t ? e : 1e3 * e) + i : l(e) ? (t = new Date(e).getTime(), isNaN(t) || t <= i ? void 0 : t) : !u(e) || (t = e.getTime()) <= i ? void 0 : t;
|
|
681
720
|
}
|
|
682
721
|
|
|
683
722
|
function T(e, t, i, r, n) {
|
|
684
|
-
var a = e.qyCache, r = (
|
|
685
|
-
return
|
|
723
|
+
var a = e.qyCache, r = (l(t) && (t = s(t)), "f" == r ? a.getOrCreateFile(t, n) : a.getOrCreateDir(t, n));
|
|
724
|
+
return q(e, "expireItem", {
|
|
725
|
+
item: r,
|
|
686
726
|
milliSecond: i
|
|
687
727
|
});
|
|
688
728
|
}
|
|
689
729
|
|
|
690
730
|
function M(e, t, i, r, n) {
|
|
691
|
-
var a = e.qyCache, i = (
|
|
692
|
-
return
|
|
731
|
+
var a = e.qyCache, i = (l(t) && (t = s(t)), "f" == i ? a.getOrCreateFile(t, n) : a.getOrCreateDir(t, n));
|
|
732
|
+
return q(e, "removeItemExpire", {
|
|
733
|
+
item: i,
|
|
693
734
|
infoFileOnly: r
|
|
694
735
|
});
|
|
695
736
|
}
|