datafolder 1.0.18 → 1.0.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -0
- package/package.json +1 -1
- package/sample.js +9 -0
- package/scripts/fastify.js +1 -0
- package/scripts/flights2.js +3 -3
- package/scripts/hnsw.js +39 -0
- package/src/QyAcl.js +56 -57
- package/src/QyBatch.d.ts +4 -2
- package/src/QyBatch.js +227 -156
- package/src/QyBinWriter.d.ts +5 -9
- package/src/QyBinWriter.js +36 -45
- package/src/QyCD.d.ts +1 -1
- package/src/QyCD.js +66 -64
- package/src/QyCache.d.ts +11 -9
- package/src/QyCache.js +125 -121
- package/src/QyDB.js +1 -4
- package/src/QyDefaultOptions.js +8 -0
- package/src/QyDir.d.ts +3 -3
- package/src/QyDir.js +33 -39
- package/src/QyEmbeddingWriter.d.ts +12 -0
- package/src/QyEmbeddingWriter.js +55 -0
- package/src/QyEmbeddings.d.ts +15 -0
- package/src/QyEmbeddings.js +128 -0
- package/src/QyFileContentUtils.js +123 -111
- package/src/QyHNSW.d.ts +5 -8
- package/src/QyHNSW.js +36 -44
- package/src/QyIndex.d.ts +1 -1
- package/src/QyIndex.js +79 -78
- package/src/QyIntersectNode.js +3 -3
- package/src/QySnapshots.js +177 -184
- package/src/QyUnionNode.d.ts +3 -3
- package/src/QyUnionNode.js +36 -36
- package/src/QyUtils.d.ts +5 -3
- package/src/QyUtils.js +130 -114
- package/static/codemirror.bundle/editor.bundle.js +16 -9
package/src/QyCache.js
CHANGED
|
@@ -1,78 +1,83 @@
|
|
|
1
1
|
import {
|
|
2
|
-
join as
|
|
2
|
+
join as v
|
|
3
3
|
} from "node:path";
|
|
4
4
|
|
|
5
5
|
import r from "node:events";
|
|
6
6
|
|
|
7
7
|
import {
|
|
8
|
-
QyDir as
|
|
8
|
+
QyDir as l
|
|
9
9
|
} from "./QyDir.js";
|
|
10
10
|
|
|
11
11
|
import {
|
|
12
|
-
QyFileLogger as
|
|
12
|
+
QyFileLogger as g
|
|
13
13
|
} from "./QyFileLogger.js";
|
|
14
14
|
|
|
15
15
|
import {
|
|
16
|
-
QyKVData as
|
|
16
|
+
QyKVData as h
|
|
17
17
|
} from "./QyKVData.js";
|
|
18
18
|
|
|
19
19
|
import {
|
|
20
|
-
QyTriggerNode as
|
|
20
|
+
QyTriggerNode as R,
|
|
21
21
|
updateTriggerNodes as s
|
|
22
22
|
} from "./QyTrigger.js";
|
|
23
23
|
|
|
24
24
|
import {
|
|
25
25
|
isString as C,
|
|
26
|
-
lockExclusiveFilePath as
|
|
27
|
-
unlockExclusiveFilePath as
|
|
26
|
+
lockExclusiveFilePath as x,
|
|
27
|
+
unlockExclusiveFilePath as o,
|
|
28
|
+
ensureDir as q,
|
|
29
|
+
SpecialFilePaths as e,
|
|
30
|
+
SpecialOperators as t
|
|
28
31
|
} from "./QyUtils.js";
|
|
29
32
|
|
|
30
33
|
import {
|
|
31
|
-
dumpStructure as
|
|
34
|
+
dumpStructure as O
|
|
32
35
|
} from "./QyStructure.js";
|
|
33
36
|
|
|
34
37
|
import {
|
|
35
38
|
getIndexPropPaths as n,
|
|
36
|
-
parseNamePathAndQuery as
|
|
39
|
+
parseNamePathAndQuery as d
|
|
37
40
|
} from "./QyIndex.js";
|
|
38
41
|
|
|
39
42
|
import {
|
|
40
|
-
getDefaultOptions as
|
|
43
|
+
getDefaultOptions as u
|
|
41
44
|
} from "./QyDefaultOptions.js";
|
|
42
45
|
|
|
43
46
|
import {
|
|
44
|
-
|
|
45
|
-
SpecialFilePaths as e
|
|
46
|
-
} from "./QyUtils.js";
|
|
47
|
-
|
|
48
|
-
import {
|
|
49
|
-
QueryCounter as l
|
|
47
|
+
QueryCounter as p
|
|
50
48
|
} from "./QyUnionNode.js";
|
|
51
49
|
|
|
52
50
|
import {
|
|
53
|
-
QyKVDataLoader as
|
|
51
|
+
QyKVDataLoader as P
|
|
54
52
|
} from "./QyKVDataLoader.js";
|
|
55
53
|
|
|
56
54
|
import {
|
|
57
|
-
requireFromModCode as
|
|
55
|
+
requireFromModCode as c
|
|
58
56
|
} from "./QyMod.js";
|
|
59
57
|
|
|
60
58
|
import {
|
|
61
|
-
logger as
|
|
59
|
+
logger as b
|
|
62
60
|
} from "./QyLogger.js";
|
|
63
61
|
|
|
62
|
+
import {
|
|
63
|
+
QyEmbeddings as D
|
|
64
|
+
} from "./QyEmbeddings.js";
|
|
65
|
+
|
|
66
|
+
var {} = t;
|
|
67
|
+
|
|
64
68
|
let {
|
|
65
69
|
HiddenFolderName: y,
|
|
66
|
-
IndexDirPath:
|
|
67
|
-
TriggerDirPath:
|
|
68
|
-
RpcDirPath:
|
|
69
|
-
ExpireDirPath:
|
|
70
|
-
MetaFilePath:
|
|
71
|
-
|
|
72
|
-
|
|
70
|
+
IndexDirPath: F,
|
|
71
|
+
TriggerDirPath: w,
|
|
72
|
+
RpcDirPath: E,
|
|
73
|
+
ExpireDirPath: m,
|
|
74
|
+
MetaFilePath: i,
|
|
75
|
+
HNSWDirPath: f,
|
|
76
|
+
InfoDirPath: Q,
|
|
77
|
+
RecycleDirPath: S
|
|
73
78
|
} = e;
|
|
74
79
|
|
|
75
|
-
class
|
|
80
|
+
class a extends r {
|
|
76
81
|
rootFolder;
|
|
77
82
|
options;
|
|
78
83
|
qyDB;
|
|
@@ -83,42 +88,35 @@ class o extends r {
|
|
|
83
88
|
rpcRootDir;
|
|
84
89
|
indexRootDir;
|
|
85
90
|
expireRootDir;
|
|
86
|
-
|
|
87
|
-
|
|
91
|
+
hnswRootDir;
|
|
92
|
+
infoRootDir;
|
|
93
|
+
recycleRootDir;
|
|
88
94
|
exclusiveFilePath;
|
|
89
95
|
qyKVData;
|
|
90
96
|
qyFileLogger;
|
|
97
|
+
qyEmbeddings;
|
|
91
98
|
rpcFunMap = {};
|
|
92
99
|
constructor(r, e, t) {
|
|
93
|
-
super(), e = {
|
|
94
|
-
...
|
|
100
|
+
super(), this.rootFolder = r, e = this.options = {
|
|
101
|
+
...u("QyCache"),
|
|
95
102
|
...e
|
|
96
|
-
},
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
[y]:
|
|
100
|
-
},
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
[n]
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
rootDir: o,
|
|
107
|
-
expireRootDir: l,
|
|
108
|
-
indexRootDir: d,
|
|
109
|
-
embeddingRootDir: h,
|
|
110
|
-
h2FRootDir: u
|
|
111
|
-
});
|
|
112
|
-
var i, o, a, s, n, g, l, d, h, u, p = f(r, "qykv");
|
|
103
|
+
}, this.qyDB = t;
|
|
104
|
+
var i, o = this, t = o.rootDir = new l("", !0, void 0, o), a = new l(y, !0, t);
|
|
105
|
+
t._subdirMap = {
|
|
106
|
+
[y]: a
|
|
107
|
+
}, a._subdirMap = {};
|
|
108
|
+
for ([ s, i ] of [ [ F, "indexRootDir" ], [ m, "expireRootDir" ], [ f, "hnswRootDir" ], [ Q, "infoRootDir" ], [ S, "recycleRootDir" ] ]) {
|
|
109
|
+
var s = s.at(-1), n = new l(s, !0, a);
|
|
110
|
+
a._subdirMap[s] = n, o[i] = n;
|
|
111
|
+
}
|
|
112
|
+
t = v(r, "qykv");
|
|
113
113
|
Object.assign(this, {
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
qyKVData: new F(p, e, this),
|
|
121
|
-
qyFileLogger: new c(f(r, "qylog"), e)
|
|
114
|
+
kvFolder: t,
|
|
115
|
+
rpcFolder: v(r, "rpc"),
|
|
116
|
+
exclusiveFilePath: v(r, ".opened"),
|
|
117
|
+
qyKVData: new h(t, e, this),
|
|
118
|
+
qyFileLogger: new g(v(r, "qylog"), e),
|
|
119
|
+
qyEmbeddings: new D(this, e)
|
|
122
120
|
});
|
|
123
121
|
}
|
|
124
122
|
async start() {
|
|
@@ -129,20 +127,21 @@ class o extends r {
|
|
|
129
127
|
qyKVData: i,
|
|
130
128
|
qyFileLogger: o,
|
|
131
129
|
exclusiveFilePath: a,
|
|
132
|
-
rootDir: s
|
|
130
|
+
rootDir: s,
|
|
131
|
+
qyEmbeddings: n
|
|
133
132
|
} = this;
|
|
134
|
-
|
|
133
|
+
b.log(`Starting at ${r}...`);
|
|
135
134
|
try {
|
|
136
135
|
await q(e), o.start();
|
|
137
136
|
var [ {
|
|
138
|
-
maxChangeId:
|
|
137
|
+
maxChangeId: l,
|
|
139
138
|
loadedMaxAclNum: g,
|
|
140
|
-
qySnapshots:
|
|
139
|
+
qySnapshots: h,
|
|
141
140
|
aclKeyValueMap: d
|
|
142
|
-
} ] = await Promise.all([ ((r, e) => (r = new
|
|
143
|
-
i.start(
|
|
141
|
+
} ] = await Promise.all([ ((r, e) => (r = new P(r, e)).load())(e, t), x(a) ]);
|
|
142
|
+
i.start(l, g, h, d), await n.start();
|
|
144
143
|
} catch (r) {
|
|
145
|
-
throw o.stop(), r;
|
|
144
|
+
throw o.stop(), n.stop(), r;
|
|
146
145
|
}
|
|
147
146
|
t.preloadFoldersAtStart && !function r(e) {
|
|
148
147
|
let {
|
|
@@ -151,48 +150,50 @@ class o extends r {
|
|
|
151
150
|
} = e;
|
|
152
151
|
for (var o of t) r(o);
|
|
153
152
|
}(s);
|
|
154
|
-
var
|
|
155
|
-
e.triggerNodes = [ new
|
|
156
|
-
for (
|
|
153
|
+
var u = this, e = u.rootDir;
|
|
154
|
+
e.triggerNodes = [ new R() ];
|
|
155
|
+
for (p of (e = u.triggerRootDir = e.searchOrCreateSubdir(w)).fileList) {
|
|
157
156
|
var {
|
|
158
|
-
name:
|
|
159
|
-
fileContent:
|
|
160
|
-
} =
|
|
161
|
-
|
|
157
|
+
name: p,
|
|
158
|
+
fileContent: c
|
|
159
|
+
} = p, [ c, D, y ] = c;
|
|
160
|
+
u.insertTriggerToTree(p, c, D, y);
|
|
162
161
|
}
|
|
163
162
|
var F = this;
|
|
164
|
-
F.rpcRootDir = F.rootDir.searchOrCreateSubdir(
|
|
163
|
+
F.rpcRootDir = F.rootDir.searchOrCreateSubdir(E);
|
|
165
164
|
for (m of F.rpcRootDir.fileList) {
|
|
166
165
|
var {
|
|
167
166
|
name: m,
|
|
168
|
-
fileContent:
|
|
167
|
+
fileContent: f
|
|
169
168
|
} = m;
|
|
170
|
-
C(
|
|
169
|
+
C(f) && F.insertRpc(m, f);
|
|
171
170
|
}
|
|
172
|
-
return t.dumpStructureAtStart && await
|
|
171
|
+
return t.dumpStructureAtStart && await O(v(r, "qystruct"), s), b.log("Started."),
|
|
173
172
|
this;
|
|
174
173
|
}
|
|
175
174
|
async stop() {
|
|
176
|
-
|
|
175
|
+
b.log("Stopping...");
|
|
177
176
|
var {
|
|
178
177
|
qyKVData: r,
|
|
179
|
-
|
|
180
|
-
|
|
178
|
+
qyEmbeddings: e,
|
|
179
|
+
qyFileLogger: t,
|
|
180
|
+
exclusiveFilePath: i
|
|
181
181
|
} = this;
|
|
182
|
-
return await r.stop(),
|
|
183
|
-
Object.assign(this, {
|
|
182
|
+
return await r.stop(), await e.stop(), b.log("QyFileLogger stopped."), await t.stop(),
|
|
183
|
+
o(i), Object.assign(this, {
|
|
184
184
|
rootDir: void 0,
|
|
185
185
|
indexRootDir: void 0,
|
|
186
186
|
expireRootDir: void 0,
|
|
187
187
|
triggerRootDir: void 0,
|
|
188
188
|
rpcRootDir: void 0,
|
|
189
|
-
|
|
190
|
-
|
|
189
|
+
hnswRootDir: void 0,
|
|
190
|
+
infoRootDir: void 0,
|
|
191
|
+
recycleRootDir: void 0,
|
|
191
192
|
qyKVData: void 0,
|
|
192
193
|
rpcFunMap: void 0,
|
|
193
194
|
qyDB: void 0,
|
|
194
195
|
qyFileLogger: void 0
|
|
195
|
-
}),
|
|
196
|
+
}), b.log("Stopped."), this;
|
|
196
197
|
}
|
|
197
198
|
increaseChangeId() {
|
|
198
199
|
return this.qyKVData.increaseChangeId();
|
|
@@ -210,20 +211,20 @@ class o extends r {
|
|
|
210
211
|
getFile(r, e = this.rootDir) {
|
|
211
212
|
return r.isFile ? r : C(r) ? e.getFile(r) : e.searchFile(r);
|
|
212
213
|
}
|
|
213
|
-
getDir(r, e = this.rootDir) {
|
|
214
|
-
return r.isDir ? r : C(r) ? e.getSubdir(r) : e.searchSubdir(r);
|
|
215
|
-
}
|
|
216
214
|
getOrCreateFile(r, e = this.rootDir) {
|
|
217
215
|
return r.isFile ? r : C(r) ? e.getOrCreateFile(r) : e.searchOrCreateFile(r);
|
|
218
216
|
}
|
|
217
|
+
getDir(r, e = this.rootDir) {
|
|
218
|
+
return r.isDir ? r : C(r) ? e.getSubdir(r) : e.searchSubdir(r);
|
|
219
|
+
}
|
|
219
220
|
getOrCreateDir(r, e = this.rootDir) {
|
|
220
221
|
return r.isDir ? r : C(r) ? e.getOrCreateSubdir(r) : e.searchOrCreateSubdir(r);
|
|
221
222
|
}
|
|
222
223
|
getIndexDir(r) {
|
|
223
|
-
return this.indexRootDir.searchSubdir(
|
|
224
|
+
return this.indexRootDir.searchSubdir(K(r.fullPathHash));
|
|
224
225
|
}
|
|
225
226
|
getOrCreateIndexDir(r) {
|
|
226
|
-
return this.indexRootDir.searchOrCreateSubdir(
|
|
227
|
+
return this.indexRootDir.searchOrCreateSubdir(K(r.fullPathHash));
|
|
227
228
|
}
|
|
228
229
|
getTriggerFile(r) {
|
|
229
230
|
return this.triggerRootDir.getFile(r);
|
|
@@ -232,29 +233,36 @@ class o extends r {
|
|
|
232
233
|
return this.triggerRootDir.getOrCreateFile(r);
|
|
233
234
|
}
|
|
234
235
|
getExpireInfoFile(r, e = "c") {
|
|
235
|
-
return this.expireRootDir.searchFile(
|
|
236
|
+
return this.expireRootDir.searchFile(L(r, e));
|
|
236
237
|
}
|
|
237
238
|
getOrCreateExpireInfoFile(r, e = "c") {
|
|
238
|
-
return this.expireRootDir.searchOrCreateFile(
|
|
239
|
+
return this.expireRootDir.searchOrCreateFile(L(r, e));
|
|
239
240
|
}
|
|
240
|
-
|
|
241
|
-
return this.
|
|
241
|
+
getInfoFile(r) {
|
|
242
|
+
return this.infoRootDir.searchFile(I(r.fullPathHash));
|
|
242
243
|
}
|
|
243
|
-
|
|
244
|
-
return this.
|
|
244
|
+
getOrCreateInfoFile(r) {
|
|
245
|
+
return this.infoRootDir.searchOrCreateFile(I(r.fullPathHash));
|
|
245
246
|
}
|
|
246
|
-
|
|
247
|
-
r = this.
|
|
248
|
-
if (r
|
|
247
|
+
hToSrcFile(r) {
|
|
248
|
+
r = this.infoRootDir.searchFile(I(r));
|
|
249
|
+
if (r && r.fileContent) {
|
|
250
|
+
r = r.fileContent.namePath;
|
|
251
|
+
if (r) return this.getOrCreateFile(r);
|
|
252
|
+
}
|
|
249
253
|
}
|
|
250
|
-
|
|
251
|
-
|
|
254
|
+
getEmbedding(r) {
|
|
255
|
+
var e, t = this.getInfoFile(r);
|
|
256
|
+
if (t && t.fileContent) return {
|
|
257
|
+
length: t,
|
|
258
|
+
idx: e
|
|
259
|
+
} = t.fileContent, this.qyEmbeddings.getEmbedding(r.fullPathHash, t, e);
|
|
252
260
|
}
|
|
253
|
-
|
|
254
|
-
return this.
|
|
261
|
+
getOrCreateRecycleFile(r) {
|
|
262
|
+
return this.recycleRootDir.getOrCreateFile(r);
|
|
255
263
|
}
|
|
256
|
-
|
|
257
|
-
return this.
|
|
264
|
+
getRecycleFile(r) {
|
|
265
|
+
return this.recycleRootDir.getFile(r);
|
|
258
266
|
}
|
|
259
267
|
get triggerFileList() {
|
|
260
268
|
return this.triggerRootDir.fileList;
|
|
@@ -269,13 +277,13 @@ class o extends r {
|
|
|
269
277
|
return this.rpcRootDir.fileList;
|
|
270
278
|
}
|
|
271
279
|
get metaFile() {
|
|
272
|
-
return this.rootDir.searchOrCreateFile(
|
|
280
|
+
return this.rootDir.searchOrCreateFile(i);
|
|
273
281
|
}
|
|
274
282
|
insertTriggerToTree(r, e, t, i) {
|
|
275
283
|
var {
|
|
276
284
|
rootDir: o,
|
|
277
285
|
qyDB: a
|
|
278
|
-
} = this, i =
|
|
286
|
+
} = this, i = c("trigger", r, i, a);
|
|
279
287
|
o.triggerNodes[0].insertTrigger(r, e, t, i), s(o, e);
|
|
280
288
|
}
|
|
281
289
|
removeTriggerFromTree(r, e) {
|
|
@@ -286,28 +294,28 @@ class o extends r {
|
|
|
286
294
|
var {
|
|
287
295
|
rpcFunMap: t,
|
|
288
296
|
qyDB: i
|
|
289
|
-
} = this, e =
|
|
297
|
+
} = this, e = c("rpc", r, e, i);
|
|
290
298
|
e && (t[r] = e);
|
|
291
299
|
}
|
|
292
300
|
removeRpc(r) {
|
|
293
301
|
delete this.rpcFunMap[r];
|
|
294
302
|
}
|
|
295
303
|
queryFiles(r, e, t, i, o = this.rootDir) {
|
|
296
|
-
var a = [], t = (C(e) && !t && (t = 1), new
|
|
297
|
-
return
|
|
304
|
+
var a = [], t = (C(e) && !t && (t = 1), new p(t, i)), i = this.options.queryAutoCreateIndex;
|
|
305
|
+
return T(r, e, a, t, o, i), a;
|
|
298
306
|
}
|
|
299
307
|
queryFilesMulti(r, e, t, i = this.rootDir) {
|
|
300
|
-
var o, a, s = [], n = new
|
|
301
|
-
for ([ o, a ] of r)
|
|
308
|
+
var o, a, s = [], n = new p(e, t), l = this.options.queryAutoCreateIndex;
|
|
309
|
+
for ([ o, a ] of r) T(o, a, s, n, i, l);
|
|
302
310
|
return s;
|
|
303
311
|
}
|
|
304
312
|
async callRpc(e, r, t) {
|
|
305
313
|
var i, o = this.rpcFunMap[e];
|
|
306
|
-
if (!o) return
|
|
314
|
+
if (!o) return b.warn(i = `RPC ${e} not implemented.`), new Error(i);
|
|
307
315
|
try {
|
|
308
316
|
return await o(r, t);
|
|
309
317
|
} catch (r) {
|
|
310
|
-
return
|
|
318
|
+
return b.error(`Calling RPC ${e} failed:`, r), r;
|
|
311
319
|
}
|
|
312
320
|
}
|
|
313
321
|
monitorItemExpire(r, e) {
|
|
@@ -324,26 +332,22 @@ class o extends r {
|
|
|
324
332
|
}
|
|
325
333
|
}
|
|
326
334
|
|
|
327
|
-
function
|
|
328
|
-
return [ r.substring(0, 2), r ];
|
|
329
|
-
}
|
|
330
|
-
|
|
331
|
-
function h(r) {
|
|
335
|
+
function I(r) {
|
|
332
336
|
return [ r.substring(0, 2), r ];
|
|
333
337
|
}
|
|
334
338
|
|
|
335
|
-
function
|
|
339
|
+
function L(r, e) {
|
|
336
340
|
return [ e, r.substring(0, 2), r ];
|
|
337
341
|
}
|
|
338
342
|
|
|
339
|
-
function
|
|
343
|
+
function K(r) {
|
|
340
344
|
return [ r[0], r ];
|
|
341
345
|
}
|
|
342
346
|
|
|
343
|
-
function
|
|
344
|
-
e && (a = a ? n(e) : void 0,
|
|
347
|
+
function T(r, e, t, i, o, a) {
|
|
348
|
+
e && (a = a ? n(e) : void 0, d(r, o, a, e, t, i));
|
|
345
349
|
}
|
|
346
350
|
|
|
347
351
|
export {
|
|
348
|
-
|
|
352
|
+
a as QyCache
|
|
349
353
|
};
|
package/src/QyDB.js
CHANGED
|
@@ -47,10 +47,7 @@ class l extends t {
|
|
|
47
47
|
...o("QyDB"),
|
|
48
48
|
...t
|
|
49
49
|
}, this.rootFolder = e;
|
|
50
|
-
for (let
|
|
51
|
-
var t = this.currentBatch;
|
|
52
|
-
return t[i].apply(t, e).run();
|
|
53
|
-
};
|
|
50
|
+
for (let t of [ "createIndex", "delIndex", "insert", "put", "remove", "delDir", "delFile", "delContent", "insertTrigger", "removeTrigger", "insertRpc", "removeRpc", "insertEmbedding", "delEmbedding", "insertWithEmbedding", "expireFile", "expireDir", "expireContent", "expireFileM", "expireDirM", "expireContentM", "removeFileExpire", "removeDirExpire", "removeContentExpire" ]) this[t] = (...e) => this.currentBatch[t](...e).run();
|
|
54
51
|
}
|
|
55
52
|
async start(e) {
|
|
56
53
|
var t;
|
package/src/QyDefaultOptions.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
var e = {
|
|
2
2
|
append: !1,
|
|
3
|
+
read: !1,
|
|
3
4
|
retryIntervalFactor: 2,
|
|
4
5
|
maxRetryInterval: 60,
|
|
5
6
|
emitSaveEvent: !1,
|
|
@@ -57,6 +58,13 @@ let t = {
|
|
|
57
58
|
},
|
|
58
59
|
QyAclWriter: {
|
|
59
60
|
minBatchSize: 2
|
|
61
|
+
},
|
|
62
|
+
QyEmbeddingWriter: {
|
|
63
|
+
append: !0,
|
|
64
|
+
read: !0
|
|
65
|
+
},
|
|
66
|
+
QyEmbeddings: {
|
|
67
|
+
appendOnly: !1
|
|
60
68
|
}
|
|
61
69
|
};
|
|
62
70
|
|
package/src/QyDir.d.ts
CHANGED
|
@@ -22,7 +22,7 @@ export declare class QyDir extends QyItem {
|
|
|
22
22
|
get subdirMap(): {};
|
|
23
23
|
get fileMap(): {};
|
|
24
24
|
getSubdir(name: string): QyDir | undefined;
|
|
25
|
-
getFile(fileName: string): QyFile | undefined;
|
|
25
|
+
getFile(fileName: string | number): QyFile | undefined;
|
|
26
26
|
getOrCreateSubdir(name: string, created?: boolean): QyDir;
|
|
27
27
|
getOrCreateFile(fileName: string, created?: boolean): QyFile;
|
|
28
28
|
searchSubdir(namePath: string[], endIdx?: number): QyDir | undefined;
|
|
@@ -43,7 +43,7 @@ export declare class QyDir extends QyItem {
|
|
|
43
43
|
allLoadedSubdirs(): any;
|
|
44
44
|
setChildTriggerNodes(child: QyDir): void;
|
|
45
45
|
filterSubdirNames(filterFun: any): Generator<QyDir, void, unknown>;
|
|
46
|
-
|
|
47
|
-
|
|
46
|
+
filterFileNames(filterFun: any): Generator<QyFile, void, unknown>;
|
|
47
|
+
compareItemRange(cmpFun: any, isFile?: boolean, reverse?: boolean): Generator<QyDir | QyFile | undefined, void, unknown>;
|
|
48
48
|
ensureAllLoaded(): this;
|
|
49
49
|
}
|
package/src/QyDir.js
CHANGED
|
@@ -6,8 +6,8 @@ import {
|
|
|
6
6
|
getSubdirMapKey as i,
|
|
7
7
|
getFileMapKey as t,
|
|
8
8
|
SpecialFilePaths as r,
|
|
9
|
-
isString as
|
|
10
|
-
getNowTime as
|
|
9
|
+
isString as h,
|
|
10
|
+
getNowTime as n
|
|
11
11
|
} from "./QyUtils.js";
|
|
12
12
|
|
|
13
13
|
import {
|
|
@@ -20,7 +20,7 @@ import {
|
|
|
20
20
|
|
|
21
21
|
let s = r.HiddenFolderName;
|
|
22
22
|
|
|
23
|
-
class
|
|
23
|
+
class d extends e {
|
|
24
24
|
_subdirCount = 0;
|
|
25
25
|
_fileCount = 0;
|
|
26
26
|
subdirMapChangeCount = 0;
|
|
@@ -43,7 +43,7 @@ class u extends e {
|
|
|
43
43
|
return f(this), this._fileCount;
|
|
44
44
|
}
|
|
45
45
|
get subdirCount() {
|
|
46
|
-
return
|
|
46
|
+
return l(this), this._subdirCount;
|
|
47
47
|
}
|
|
48
48
|
get isDir() {
|
|
49
49
|
return !0;
|
|
@@ -61,7 +61,7 @@ class u extends e {
|
|
|
61
61
|
return this._fileMapKey || (this._fileMapKey = t(this.fullPathHash));
|
|
62
62
|
}
|
|
63
63
|
get subdirMap() {
|
|
64
|
-
return
|
|
64
|
+
return l(this), this._subdirMap;
|
|
65
65
|
}
|
|
66
66
|
get fileMap() {
|
|
67
67
|
return f(this), this._fileMap;
|
|
@@ -84,7 +84,7 @@ class u extends e {
|
|
|
84
84
|
if ("/" === e) return this.qyCache?.rootDir || this;
|
|
85
85
|
let t = this.subdirMap;
|
|
86
86
|
var r = (t = t || (this._subdirMap = {}))[e];
|
|
87
|
-
return r || (r = t[e] = new
|
|
87
|
+
return r || (r = t[e] = new d(e, i, this), this.setChildTriggerNodes(r)),
|
|
88
88
|
r;
|
|
89
89
|
}
|
|
90
90
|
getOrCreateFile(e, i = !1) {
|
|
@@ -117,28 +117,28 @@ class u extends e {
|
|
|
117
117
|
return this.searchOrCreateSubdir(e, i, e.length - 1).getOrCreateFile(e.at(-1), i);
|
|
118
118
|
}
|
|
119
119
|
get subdirList() {
|
|
120
|
-
return
|
|
120
|
+
return u(this, "_subdirList", this.subdirMapChangeCount, () => this.created && this.subdirMap ? Object.values(this.subdirMap).filter(e => e.created) : []);
|
|
121
121
|
}
|
|
122
122
|
get subdirNameList() {
|
|
123
|
-
return
|
|
123
|
+
return u(this, "_subdirNameList", this.subdirMapChangeCount, () => this.subdirList.map(e => e.name));
|
|
124
124
|
}
|
|
125
125
|
get sortedSubdirList() {
|
|
126
|
-
return
|
|
126
|
+
return u(this, "_sortedSubdirList", this.subdirMapChangeCount, () => this.subdirList.sort((e, i) => e.cmp(i)));
|
|
127
127
|
}
|
|
128
128
|
get sortedSubdirNameList() {
|
|
129
|
-
return
|
|
129
|
+
return u(this, "_sortedSubdirNameList", this.subdirMapChangeCount, () => this.sortedSubdirList.map(e => e.nameNum));
|
|
130
130
|
}
|
|
131
131
|
get fileList() {
|
|
132
|
-
return
|
|
132
|
+
return u(this, "_fileList", this.fileMapChangeCount, () => this.created && this.fileMap ? Object.values(this.fileMap).filter(e => e.created) : []);
|
|
133
133
|
}
|
|
134
134
|
get fileNameList() {
|
|
135
|
-
return
|
|
135
|
+
return u(this, "_fileNameList", this.fileMapChangeCount, () => this.fileList.map(e => e.name));
|
|
136
136
|
}
|
|
137
137
|
get sortedFileList() {
|
|
138
|
-
return
|
|
138
|
+
return u(this, "_sortedFileList", this.fileMapChangeCount, () => this.fileList.sort((e, i) => e.cmp(i)));
|
|
139
139
|
}
|
|
140
140
|
get sortedFileNameList() {
|
|
141
|
-
return
|
|
141
|
+
return u(this, "_sortedFileNameList", this.fileMapChangeCount, () => this.sortedFileList.map(e => e.nameNum));
|
|
142
142
|
}
|
|
143
143
|
*allSubdirs() {
|
|
144
144
|
var e, i, t = this.subdirList;
|
|
@@ -172,24 +172,18 @@ class u extends e {
|
|
|
172
172
|
*filterSubdirNames(e) {
|
|
173
173
|
for (var i of this.subdirList) e(i.name) && (yield i);
|
|
174
174
|
}
|
|
175
|
-
*
|
|
176
|
-
var i
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
let r = !1;
|
|
188
|
-
for (let e = t.length - 1; 0 <= e; --e) {
|
|
189
|
-
var s = t[e], a = s.nameNum;
|
|
190
|
-
if (d(a)) {
|
|
191
|
-
if (r) return;
|
|
192
|
-
} else if (i(a)) yield s, r = !0; else if (r) return;
|
|
175
|
+
*filterFileNames(e) {
|
|
176
|
+
for (var i of this.fileList) e(i.name) && (yield i);
|
|
177
|
+
}
|
|
178
|
+
*compareItemRange(i, e = !1, t = !1) {
|
|
179
|
+
var r = e ? this.sortedFileList : this.sortedSubdirList;
|
|
180
|
+
let s = !1;
|
|
181
|
+
var a = r.length;
|
|
182
|
+
for (let e = t ? a - 1 : 0; t ? 0 <= e : e < a; t ? --e : ++e) {
|
|
183
|
+
var d = r[e], u = d.nameNum;
|
|
184
|
+
if (h(u)) {
|
|
185
|
+
if (s) return;
|
|
186
|
+
} else if (i(u)) yield d, s = !0; else if (s) return;
|
|
193
187
|
}
|
|
194
188
|
}
|
|
195
189
|
ensureAllLoaded() {
|
|
@@ -198,12 +192,12 @@ class u extends e {
|
|
|
198
192
|
}
|
|
199
193
|
}
|
|
200
194
|
|
|
201
|
-
function
|
|
195
|
+
function u(e, i, t, r) {
|
|
202
196
|
var e = e.cache, s = i + "ChangeCount";
|
|
203
197
|
return e[i] && e[s] == t || (e[i] = r(), e[s] = t), e[i];
|
|
204
198
|
}
|
|
205
199
|
|
|
206
|
-
function
|
|
200
|
+
function l(e) {
|
|
207
201
|
if (e.created && !e.subdirMapLoaded) {
|
|
208
202
|
e.subdirMapLoaded = !0;
|
|
209
203
|
var i = e.qyCache;
|
|
@@ -211,7 +205,7 @@ function n(e) {
|
|
|
211
205
|
var t = i.qyKVData.getValueSync(e.subdirMapKey);
|
|
212
206
|
if (t) {
|
|
213
207
|
var r, s = e._subdirMap || (e._subdirMap = {});
|
|
214
|
-
for (r in t) r in s || (s[r] = new
|
|
208
|
+
for (r in t) r in s || (s[r] = new d(r, !0, e));
|
|
215
209
|
e.underHiddenFolder || p(e, i, "d");
|
|
216
210
|
}
|
|
217
211
|
}
|
|
@@ -243,14 +237,14 @@ function p(e, i, t) {
|
|
|
243
237
|
}
|
|
244
238
|
var r = i.getExpireInfoFile(e.fullPathHash, t);
|
|
245
239
|
if (r) {
|
|
246
|
-
var s, a = r.fileContent, d =
|
|
240
|
+
var s, a = r.fileContent, d = n(), u = i.qyDB;
|
|
247
241
|
for (s in a) {
|
|
248
|
-
var h = "d" == t ? e.getOrCreateSubdir(s) : e.getOrCreateFile(s),
|
|
249
|
-
|
|
242
|
+
var h = "d" == t ? e.getOrCreateSubdir(s) : e.getOrCreateFile(s), l = a[s];
|
|
243
|
+
l <= d ? o(u, h) : i.monitorItemExpire(h, l);
|
|
250
244
|
}
|
|
251
245
|
}
|
|
252
246
|
}
|
|
253
247
|
|
|
254
248
|
export {
|
|
255
|
-
|
|
249
|
+
d as QyDir
|
|
256
250
|
};
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { QyBinWriter } from "./QyBinWriter.ts";
|
|
2
|
+
export declare class QyEmbeddingWriter extends QyBinWriter {
|
|
3
|
+
length: any;
|
|
4
|
+
byteLength: any;
|
|
5
|
+
totalCount: number;
|
|
6
|
+
constructor(filePath: any | undefined, length: any);
|
|
7
|
+
start(): Promise<void>;
|
|
8
|
+
append(embedding: Float32Array, needPromise?: any): any;
|
|
9
|
+
pWrite(embedding: Float32Array, idx: any, needPromise?: any): any;
|
|
10
|
+
_op_pWrite(buffer: any, idx: any): any;
|
|
11
|
+
readSync(idx: any): Float32Array<ArrayBuffer> | undefined;
|
|
12
|
+
}
|